Merge pull request #391 from MauroDataMapper/feature/gh-354

Add batching to potentially large deletion operations
MauroDataMapper · Feb 10, 2023 · 7a57b73 · 7a57b73
2 parents 30a19bd + 2e0307f
commit 7a57b73
Show file tree

Hide file tree

Showing 3 changed files with 43 additions and 22 deletions.
diff --git a/mdm-common/src/main/groovy/uk/ac/ox/softeng/maurodatamapper/util/Utils.groovy b/mdm-common/src/main/groovy/uk/ac/ox/softeng/maurodatamapper/util/Utils.groovy
@@ -215,4 +215,16 @@ class Utils {
             Thread.currentThread().interrupt()
         }
     }
+
+    /*
+        Batching: for partitioning collections into sets of smaller ones
+     */
+    static List<List> partition(List inputList, int partitionSize ) {
+        List<List> partitions = []
+        for (int i = 0; i < inputList.size(); i += partitionSize) {
+            partitions.add(inputList.subList(i, Math.min(i + partitionSize, inputList.size())));
+        }
+        return partitions
+    }
+
 }
diff --git a/...ls-app/services/uk/ac/ox/softeng/maurodatamapper/datamodel/item/DataElementService.groovy b/...ls-app/services/uk/ac/ox/softeng/maurodatamapper/datamodel/item/DataElementService.groovy
@@ -116,15 +116,20 @@ class DataElementService extends ModelItemService<DataElement> implements Summar
         }.id().list() as List<UUID>
 
         if (dataElementIds) {
-            log.trace('Removing facets for {} DataElements', dataElementIds.size())
-            deleteAllFacetsByMultiFacetAwareIds(dataElementIds,
-                                                'delete from datamodel.join_dataelement_to_facet where dataelement_id in :ids')
-
-            log.trace('Removing {} DataElements', dataElementIds.size())
-            sessionFactory.currentSession
-                .createSQLQuery('DELETE FROM datamodel.data_element WHERE id IN :ids')
-                .setParameter('ids', dataElementIds)
-                .executeUpdate()
+            List<List<UUID>> batches = Utils.partition(dataElementIds, BATCH_SIZE)
+            batches.each { ids ->
+
+                log.trace('Removing facets for {} DataElements', ids.size())
+
+                deleteAllFacetsByMultiFacetAwareIds(ids,
+                                                    'delete from datamodel.join_dataelement_to_facet where dataelement_id in :ids')
+
+                log.trace('Removing {} DataElements', ids.size())
+                sessionFactory.currentSession
+                    .createSQLQuery('DELETE FROM datamodel.data_element WHERE id IN :ids')
+                    .setParameter('ids', ids)
+                    .executeUpdate()
+            }
         }
         log.trace('DataElements removed')
     }

diff --git a/.../grails-app/services/uk/ac/ox/softeng/maurodatamapper/terminology/item/TermService.groovy b/.../grails-app/services/uk/ac/ox/softeng/maurodatamapper/terminology/item/TermService.groovy
@@ -102,24 +102,28 @@ class TermService extends ModelItemService<Term> {
         List<UUID> termIds = Term.byTerminologyIdInList(modelIds).id().list() as List<UUID>
 
         if (termIds) {
-            log.trace('Removing TermRelationships in {} Terms', termIds.size())
-            termRelationshipService.deleteAllByModelIds(modelIds)
+            List<List<UUID>> batches = Utils.partition(termIds, BATCH_SIZE)
+            batches.each { batchedIds ->
 
-            log.trace('Removing CodeSet references for {} Terms', termIds.size())
-            sessionFactory.currentSession
-                .createSQLQuery('DELETE FROM terminology.join_codeset_to_term WHERE term_id IN :ids')
-                .setParameter('ids', termIds)
-                .executeUpdate()
+                log.trace('Removing TermRelationships in {} Terms', batchedIds.size())
+                termRelationshipService.deleteAllByModelIds(modelIds)
+
+                log.trace('Removing CodeSet references for {} Terms', batchedIds.size())
+                sessionFactory.currentSession
+                        .createSQLQuery('DELETE FROM terminology.join_codeset_to_term WHERE term_id IN :ids')
+                        .setParameter('ids', batchedIds)
+                        .executeUpdate()
 
-            log.trace('Removing facets for {} Terms', termIds.size())
-            deleteAllFacetsByMultiFacetAwareIds(termIds,
-                                                'delete from terminology.join_term_to_facet where term_id in :ids')
+                log.trace('Removing facets for {} Terms', batchedIds.size())
+                deleteAllFacetsByMultiFacetAwareIds(batchedIds,
+                        'delete from terminology.join_term_to_facet where term_id in :ids')
 
+            }
             log.trace('Removing {} Terms', termIds.size())
             sessionFactory.currentSession
-                .createSQLQuery('DELETE FROM terminology.term WHERE terminology_id IN :ids')
-                .setParameter('ids', modelIds)
-                .executeUpdate()
+                    .createSQLQuery('DELETE FROM terminology.term WHERE terminology_id IN :ids')
+                    .setParameter('ids', modelIds)
+                    .executeUpdate()
 
             log.trace('Terms removed')
         }