Memory Leak Issue

[dedenbangkit]

wai-sdg-portal-worker-1  | Filename: ./tasks/validation.py
wai-sdg-portal-worker-1  | 
wai-sdg-portal-worker-1  | Line #    Mem usage    Increment  Occurrences   Line Contents
wai-sdg-portal-worker-1  | =============================================================
wai-sdg-portal-worker-1  |    243    115.7 MiB    115.7 MiB           1   @profile
wai-sdg-portal-worker-1  |    244                                         def validate(session: Session, form: int, administration: int, file: str):
wai-sdg-portal-worker-1  |    245    115.7 MiB      0.0 MiB           1       try:
wai-sdg-portal-worker-1  |    246    124.1 MiB      8.4 MiB           1           sheet_names = validate_sheet_name(file)
wai-sdg-portal-worker-1  |    247    124.1 MiB      0.0 MiB           1           template_sheets = ["data", "definitions", "administration"]
wai-sdg-portal-worker-1  |    248    124.1 MiB      0.0 MiB           1           TESTING = os.environ.get("TESTING")
wai-sdg-portal-worker-1  |    249    124.1 MiB      0.0 MiB           1           if TESTING:
wai-sdg-portal-worker-1  |    250                                                     template_sheets = ["data"]
wai-sdg-portal-worker-1  |    251    124.1 MiB      0.0 MiB           4           for sheet_tab in template_sheets:
wai-sdg-portal-worker-1  |    252    124.1 MiB      0.0 MiB           3               if sheet_tab not in sheet_names:
wai-sdg-portal-worker-1  |    253                                                         return [
wai-sdg-portal-worker-1  |    254                                                             {
wai-sdg-portal-worker-1  |    255                                                                 "error": ExcelError.sheet,
wai-sdg-portal-worker-1  |    256                                                                 "error_message": ValidationText.template_validation.value,
wai-sdg-portal-worker-1  |    257                                                                 "sheets": ",".join(sheet_names),
wai-sdg-portal-worker-1  |    258                                                             }
wai-sdg-portal-worker-1  |    259                                                         ]
wai-sdg-portal-worker-1  |    260    124.1 MiB      0.0 MiB           1           questions = crud_question.get_excel_question(session=session, form=form)
wai-sdg-portal-worker-1  |    261    124.3 MiB      0.2 MiB          23           header_names = [q.to_excel_header for q in questions.all()]
wai-sdg-portal-worker-1  |    262    126.0 MiB      1.6 MiB           1           df = pd.read_excel(file, sheet_name="data")
wai-sdg-portal-worker-1  |    263    126.0 MiB      0.0 MiB           1           if df.shape[0] == 0:
wai-sdg-portal-worker-1  |    264                                                     return [
wai-sdg-portal-worker-1  |    265                                                         {
wai-sdg-portal-worker-1  |    266                                                             "error": ExcelError.sheet,
wai-sdg-portal-worker-1  |    267                                                             "error_message": ValidationText.file_empty_validation.value,
wai-sdg-portal-worker-1  |    268                                                         }
wai-sdg-portal-worker-1  |    269                                                     ]
wai-sdg-portal-worker-1  |    270    126.0 MiB      0.0 MiB           1           excel_head = {}
wai-sdg-portal-worker-1  |    271    126.0 MiB      0.0 MiB           1           excel_cols = list(itertools.islice(generate_excel_columns(), df.shape[1]))
wai-sdg-portal-worker-1  |    272    126.0 MiB      0.0 MiB          21           for index, header in enumerate(list(df)):
wai-sdg-portal-worker-1  |    273    126.0 MiB      0.0 MiB          20               excel_head.update({excel_cols[index]: header})
wai-sdg-portal-worker-1  |    274    126.0 MiB      0.0 MiB           1           header_error = []
wai-sdg-portal-worker-1  |    275    126.0 MiB      0.0 MiB           1           data_error = []
wai-sdg-portal-worker-1  |    276    126.0 MiB      0.0 MiB           2           childs = crud_administration.get_all_childs(
wai-sdg-portal-worker-1  |    277    126.0 MiB      0.0 MiB           1               session=session, parents=[administration], current=[]
wai-sdg-portal-worker-1  |    278                                                 )
wai-sdg-portal-worker-1  |    279    126.0 MiB      0.0 MiB           2           adm = crud_administration.get_administration_by_id(
wai-sdg-portal-worker-1  |    280    126.0 MiB      0.0 MiB           1               session=session, id=administration
wai-sdg-portal-worker-1  |    281                                                 )
wai-sdg-portal-worker-1  |    282    126.0 MiB      0.0 MiB           1           adm = {"id": adm.id, "name": adm.name, "childs": childs}
wai-sdg-portal-worker-1  |    283    126.0 MiB      0.0 MiB           1           answered = []
wai-sdg-portal-worker-1  |    284    131.3 MiB      0.0 MiB          21           for col in excel_head:
wai-sdg-portal-worker-1  |    285    131.0 MiB      0.0 MiB          20               header = excel_head[col]
wai-sdg-portal-worker-1  |    286    131.0 MiB      0.0 MiB          20               error = validate_header_names(header, f"{col}1", header_names)
wai-sdg-portal-worker-1  |    287    131.0 MiB      0.0 MiB          20               if error:
wai-sdg-portal-worker-1  |    288                                                         header_error.append(error)
wai-sdg-portal-worker-1  |    289    131.0 MiB      0.0 MiB          20               if not error:
wai-sdg-portal-worker-1  |    290    131.0 MiB      0.0 MiB          20                   qid = header.split("|")[0]
wai-sdg-portal-worker-1  |    291    131.0 MiB      0.0 MiB          20                   question = questions.filter(Question.id == int(qid)).first()
wai-sdg-portal-worker-1  |    292    131.0 MiB      0.0 MiB          20                   answers = list(df[header])
wai-sdg-portal-worker-1  |    293    131.3 MiB      0.0 MiB       17820                   for i, answer in enumerate(answers):
wai-sdg-portal-worker-1  |    294    131.3 MiB      0.0 MiB       17800                       ix = i + 2
wai-sdg-portal-worker-1  |    295    131.3 MiB      0.5 MiB       17800                       valid_deps = False
wai-sdg-portal-worker-1  |    296    131.3 MiB      0.0 MiB       17800                       answer_deps = None
wai-sdg-portal-worker-1  |    297    131.3 MiB      0.0 MiB       35600                       answered.append(
wai-sdg-portal-worker-1  |    298    131.3 MiB      0.5 MiB       17800                           {
wai-sdg-portal-worker-1  |    299    131.3 MiB      0.0 MiB       17800                               "id": question.id,
wai-sdg-portal-worker-1  |    300    131.3 MiB      0.0 MiB       17800                               "answer": answer,
wai-sdg-portal-worker-1  |    301    131.3 MiB      0.2 MiB       17800                               "cell": f"{col}{ix}",
wai-sdg-portal-worker-1  |    302    131.3 MiB      0.0 MiB       17800                               "index": ix,
wai-sdg-portal-worker-1  |    303                                                                 }
wai-sdg-portal-worker-1  |    304                                                             )
wai-sdg-portal-worker-1  |    305    131.3 MiB      0.0 MiB       17800                       if question.dependency:
wai-sdg-portal-worker-1  |    306    131.3 MiB      0.2 MiB       19580                           valid_deps, answer_deps = dependency_checker(
wai-sdg-portal-worker-1  |    307    131.3 MiB      0.0 MiB        9790                               qs=question.dependency, answered=answered, index=ix
wai-sdg-portal-worker-1  |    308                                                                 )
wai-sdg-portal-worker-1  |    309    131.3 MiB      3.1 MiB       35600                       error = validate_row_data(
wai-sdg-portal-worker-1  |    310    131.3 MiB      0.0 MiB       17800                           session,
wai-sdg-portal-worker-1  |    311    131.3 MiB      0.8 MiB       17800                           f"{col}{ix}",
wai-sdg-portal-worker-1  |    312    131.3 MiB      0.0 MiB       17800                           answer,
wai-sdg-portal-worker-1  |    313    131.3 MiB      0.0 MiB       17800                           question,
wai-sdg-portal-worker-1  |    314    131.3 MiB      0.0 MiB       17800                           adm,
wai-sdg-portal-worker-1  |    315    131.3 MiB      0.0 MiB       17800                           valid_deps,
wai-sdg-portal-worker-1  |    316    131.3 MiB      0.0 MiB       17800                           answer_deps,
wai-sdg-portal-worker-1  |    317                                                             )
wai-sdg-portal-worker-1  |    318    131.3 MiB      0.0 MiB       17800                       if error:
wai-sdg-portal-worker-1  |    319                                                                 data_error.append(error)
wai-sdg-portal-worker-1  |    320    131.3 MiB      0.0 MiB           1           return header_error + data_error
wai-sdg-portal-worker-1  |    321                                             except Exception as e:
wai-sdg-portal-worker-1  |    322                                                 print("VALIDATION ERROR", str(e))
wai-sdg-portal-worker-1  |    323                                                 return None