From 2e0307f53ef65ae9cba4f3c7b3a5ede5f35e1753 Mon Sep 17 00:00:00 2001
From: James Welch <james.welch@cs.ox.ac.uk>
Date: Fri, 3 Feb 2023 16:03:27 +0000
Subject: [PATCH] Add batching to potentially large deletion operations

---
 .../softeng/maurodatamapper/util/Utils.groovy | 12 ++++++++
 .../datamodel/item/DataElementService.groovy  | 23 ++++++++------
 .../terminology/item/TermService.groovy       | 30 +++++++++++--------
 3 files changed, 43 insertions(+), 22 deletions(-)

diff --git a/mdm-common/src/main/groovy/uk/ac/ox/softeng/maurodatamapper/util/Utils.groovy b/mdm-common/src/main/groovy/uk/ac/ox/softeng/maurodatamapper/util/Utils.groovy
index 9674c2722d..cffc11a9ef 100644
--- a/mdm-common/src/main/groovy/uk/ac/ox/softeng/maurodatamapper/util/Utils.groovy
+++ b/mdm-common/src/main/groovy/uk/ac/ox/softeng/maurodatamapper/util/Utils.groovy
@@ -215,4 +215,16 @@ class Utils {
             Thread.currentThread().interrupt()
         }
     }
+
+    /*
+        Batching: for partitioning collections into sets of smaller ones
+     */
+    static List<List> partition(List inputList, int partitionSize ) {
+        List<List> partitions = []
+        for (int i = 0; i < inputList.size(); i += partitionSize) {
+            partitions.add(inputList.subList(i, Math.min(i + partitionSize, inputList.size())));
+        }
+        return partitions
+    }
+
 }
diff --git a/mdm-plugin-datamodel/grails-app/services/uk/ac/ox/softeng/maurodatamapper/datamodel/item/DataElementService.groovy b/mdm-plugin-datamodel/grails-app/services/uk/ac/ox/softeng/maurodatamapper/datamodel/item/DataElementService.groovy
index df853d3726..abb009dec4 100644
--- a/mdm-plugin-datamodel/grails-app/services/uk/ac/ox/softeng/maurodatamapper/datamodel/item/DataElementService.groovy
+++ b/mdm-plugin-datamodel/grails-app/services/uk/ac/ox/softeng/maurodatamapper/datamodel/item/DataElementService.groovy
@@ -116,15 +116,20 @@ class DataElementService extends ModelItemService<DataElement> implements Summar
         }.id().list() as List<UUID>
 
         if (dataElementIds) {
-            log.trace('Removing facets for {} DataElements', dataElementIds.size())
-            deleteAllFacetsByMultiFacetAwareIds(dataElementIds,
-                                                'delete from datamodel.join_dataelement_to_facet where dataelement_id in :ids')
-
-            log.trace('Removing {} DataElements', dataElementIds.size())
-            sessionFactory.currentSession
-                .createSQLQuery('DELETE FROM datamodel.data_element WHERE id IN :ids')
-                .setParameter('ids', dataElementIds)
-                .executeUpdate()
+            List<List<UUID>> batches = Utils.partition(dataElementIds, BATCH_SIZE)
+            batches.each { ids ->
+
+                log.trace('Removing facets for {} DataElements', ids.size())
+
+                deleteAllFacetsByMultiFacetAwareIds(ids,
+                                                    'delete from datamodel.join_dataelement_to_facet where dataelement_id in :ids')
+
+                log.trace('Removing {} DataElements', ids.size())
+                sessionFactory.currentSession
+                    .createSQLQuery('DELETE FROM datamodel.data_element WHERE id IN :ids')
+                    .setParameter('ids', ids)
+                    .executeUpdate()
+            }
         }
         log.trace('DataElements removed')
     }
diff --git a/mdm-plugin-terminology/grails-app/services/uk/ac/ox/softeng/maurodatamapper/terminology/item/TermService.groovy b/mdm-plugin-terminology/grails-app/services/uk/ac/ox/softeng/maurodatamapper/terminology/item/TermService.groovy
index b05a64b48f..f15cb60427 100644
--- a/mdm-plugin-terminology/grails-app/services/uk/ac/ox/softeng/maurodatamapper/terminology/item/TermService.groovy
+++ b/mdm-plugin-terminology/grails-app/services/uk/ac/ox/softeng/maurodatamapper/terminology/item/TermService.groovy
@@ -102,24 +102,28 @@ class TermService extends ModelItemService<Term> {
         List<UUID> termIds = Term.byTerminologyIdInList(modelIds).id().list() as List<UUID>
 
         if (termIds) {
-            log.trace('Removing TermRelationships in {} Terms', termIds.size())
-            termRelationshipService.deleteAllByModelIds(modelIds)
+            List<List<UUID>> batches = Utils.partition(termIds, BATCH_SIZE)
+            batches.each { batchedIds ->
 
-            log.trace('Removing CodeSet references for {} Terms', termIds.size())
-            sessionFactory.currentSession
-                .createSQLQuery('DELETE FROM terminology.join_codeset_to_term WHERE term_id IN :ids')
-                .setParameter('ids', termIds)
-                .executeUpdate()
+                log.trace('Removing TermRelationships in {} Terms', batchedIds.size())
+                termRelationshipService.deleteAllByModelIds(modelIds)
+
+                log.trace('Removing CodeSet references for {} Terms', batchedIds.size())
+                sessionFactory.currentSession
+                        .createSQLQuery('DELETE FROM terminology.join_codeset_to_term WHERE term_id IN :ids')
+                        .setParameter('ids', batchedIds)
+                        .executeUpdate()
 
-            log.trace('Removing facets for {} Terms', termIds.size())
-            deleteAllFacetsByMultiFacetAwareIds(termIds,
-                                                'delete from terminology.join_term_to_facet where term_id in :ids')
+                log.trace('Removing facets for {} Terms', batchedIds.size())
+                deleteAllFacetsByMultiFacetAwareIds(batchedIds,
+                        'delete from terminology.join_term_to_facet where term_id in :ids')
 
+            }
             log.trace('Removing {} Terms', termIds.size())
             sessionFactory.currentSession
-                .createSQLQuery('DELETE FROM terminology.term WHERE terminology_id IN :ids')
-                .setParameter('ids', modelIds)
-                .executeUpdate()
+                    .createSQLQuery('DELETE FROM terminology.term WHERE terminology_id IN :ids')
+                    .setParameter('ids', modelIds)
+                    .executeUpdate()
 
             log.trace('Terms removed')
         }