Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

Add batching to potentially large deletion operations #391

Merged
merged 1 commit into from Feb 10, 2023
Merged

Add batching to potentially large deletion operations #391

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Jump to
Jump to file
Failed to load files.
Diff view
Diff view
12 changes: 12 additions & 0 deletions mdm-common/src/main/groovy/uk/ac/ox/softeng/maurodatamapper/util/Utils.groovy
View file
Open in desktop
Expand Up @@ -215,4 +215,16 @@ class Utils {
Thread.currentThread().interrupt()
}
}

/*
Batching: for partitioning collections into sets of smaller ones
*/
static List<List> partition(List inputList, int partitionSize ) {
List<List> partitions = []
for (int i = 0; i < inputList.size(); i += partitionSize) {
partitions.add(inputList.subList(i, Math.min(i + partitionSize, inputList.size())));
}
return partitions
}

}
23 changes: 14 additions & 9 deletions ...ls-app/services/uk/ac/ox/softeng/maurodatamapper/datamodel/item/DataElementService.groovy
View file
Open in desktop
Expand Up @@ -116,15 +116,20 @@ class DataElementService extends ModelItemService<DataElement> implements Summar
}.id().list() as List<UUID>

if (dataElementIds) {
log.trace('Removing facets for {} DataElements', dataElementIds.size())
deleteAllFacetsByMultiFacetAwareIds(dataElementIds,
'delete from datamodel.join_dataelement_to_facet where dataelement_id in :ids')

log.trace('Removing {} DataElements', dataElementIds.size())
sessionFactory.currentSession
.createSQLQuery('DELETE FROM datamodel.data_element WHERE id IN :ids')
.setParameter('ids', dataElementIds)
.executeUpdate()
List<List<UUID>> batches = Utils.partition(dataElementIds, BATCH_SIZE)
batches.each { ids ->

log.trace('Removing facets for {} DataElements', ids.size())

deleteAllFacetsByMultiFacetAwareIds(ids,
'delete from datamodel.join_dataelement_to_facet where dataelement_id in :ids')

log.trace('Removing {} DataElements', ids.size())
sessionFactory.currentSession
.createSQLQuery('DELETE FROM datamodel.data_element WHERE id IN :ids')
.setParameter('ids', ids)
.executeUpdate()
}
}
log.trace('DataElements removed')
}
Expand Down
30 changes: 17 additions & 13 deletions .../grails-app/services/uk/ac/ox/softeng/maurodatamapper/terminology/item/TermService.groovy
View file
Open in desktop
Expand Up @@ -102,24 +102,28 @@ class TermService extends ModelItemService<Term> {
List<UUID> termIds = Term.byTerminologyIdInList(modelIds).id().list() as List<UUID>

if (termIds) {
log.trace('Removing TermRelationships in {} Terms', termIds.size())
termRelationshipService.deleteAllByModelIds(modelIds)
List<List<UUID>> batches = Utils.partition(termIds, BATCH_SIZE)
batches.each { batchedIds ->

log.trace('Removing CodeSet references for {} Terms', termIds.size())
sessionFactory.currentSession
.createSQLQuery('DELETE FROM terminology.join_codeset_to_term WHERE term_id IN :ids')
.setParameter('ids', termIds)
.executeUpdate()
log.trace('Removing TermRelationships in {} Terms', batchedIds.size())
termRelationshipService.deleteAllByModelIds(modelIds)

log.trace('Removing CodeSet references for {} Terms', batchedIds.size())
sessionFactory.currentSession
.createSQLQuery('DELETE FROM terminology.join_codeset_to_term WHERE term_id IN :ids')
.setParameter('ids', batchedIds)
.executeUpdate()

log.trace('Removing facets for {} Terms', termIds.size())
deleteAllFacetsByMultiFacetAwareIds(termIds,
'delete from terminology.join_term_to_facet where term_id in :ids')
log.trace('Removing facets for {} Terms', batchedIds.size())
deleteAllFacetsByMultiFacetAwareIds(batchedIds,
'delete from terminology.join_term_to_facet where term_id in :ids')

}
log.trace('Removing {} Terms', termIds.size())
sessionFactory.currentSession
.createSQLQuery('DELETE FROM terminology.term WHERE terminology_id IN :ids')
.setParameter('ids', modelIds)
.executeUpdate()
.createSQLQuery('DELETE FROM terminology.term WHERE terminology_id IN :ids')
.setParameter('ids', modelIds)
.executeUpdate()

log.trace('Terms removed')
}
Expand Down