From bf7aaa12f237c5e1a6994e0adba913254ecc82af Mon Sep 17 00:00:00 2001 From: vangef Date: Fri, 23 Feb 2024 21:23:23 +0000 Subject: [PATCH] terminal output improvements + flush=True for print() --- organise_gradebook.py | 5 +++-- utils/extractor.py | 10 +++++----- utils/inspector.py | 14 +++++++------- utils/organiser.py | 18 +++++++++--------- 4 files changed, 24 insertions(+), 23 deletions(-) diff --git a/organise_gradebook.py b/organise_gradebook.py index d9b956f..b59034e 100644 --- a/organise_gradebook.py +++ b/organise_gradebook.py @@ -1,14 +1,15 @@ import os, sys from utils.organiser import organise_gradebook, check_submissions_dir_for_compressed + def main(): gradebook_name = ' '.join(sys.argv[1:]) if len(sys.argv) > 1 else exit(f'\nNo gradebook name given. Provide the name as an argument.\n\nUsage: python {sys.argv[0]} [gradebook dir name]\n') gradebook_dir = os.path.join('BB_gradebooks', gradebook_name) # gradebook from Blackboard with all submissions submissions_dir = os.path.join('BB_submissions', gradebook_name) # target dir for extracted submissions abs_path = os.getcwd() # absolute path of main/this script - print(f'\nGradebook directory to organise: {os.path.join(abs_path, gradebook_dir)}') - + print(f'\nGradebook directory to organise:\n{os.path.join(abs_path, gradebook_dir)}', flush=True) + organise_gradebook(gradebook_dir, submissions_dir) check_submissions_dir_for_compressed(submissions_dir) diff --git a/utils/extractor.py b/utils/extractor.py index b4089ab..f255f14 100644 --- a/utils/extractor.py +++ b/utils/extractor.py @@ -11,9 +11,9 @@ def mark_file_as_BAD(file: str, bad_exception: Exception) -> None: os.makedirs(bad_dir, exist_ok=True) bad_file_path = os.path.join(bad_dir, filename) shutil.move(file, bad_file_path) - print(f'\n[Warning] Found BAD compressed file: {filename}\nMoved to: {bad_file_path}\nError message: {bad_exception}') + print(f'\n[Warning] Found BAD compressed file: {filename}\nMoved to: {bad_file_path}\nError message: {bad_exception}\n', flush=True) except Exception as e: - print(f'\n[Error] {e}') + print(f'\n[ERROR] {e}\n', flush=True) def extract_zip(zip_file: str, target_dir: str) -> None | Exception: try: @@ -24,7 +24,7 @@ def extract_zip(zip_file: str, target_dir: str) -> None | Exception: except zipfile.BadZipfile as e: mark_file_as_BAD(zip_file, e) except Exception as e: - print(f'\n[ERROR] Something went wrong while extracting the contents of a submitted zip file. Check the error message, get student id and download / organise manually\nError message: {e}') + print(f'\n[ERROR] Something went wrong while extracting the contents of a submitted zip file. Check the error message, get student id and download / organise manually\n\nError message: {e}\n', flush=True) return e def extract_rar(rar_file: str, target_dir: str) -> None: @@ -45,7 +45,7 @@ def extract_rar(rar_file: str, target_dir: str) -> None: except rarfile.NotRarFile as e: mark_file_as_BAD(rar_file, e) except rarfile.RarCannotExec as e: - print('\n[Error] Missing unrar tool\nfor Windows: make sure file UnRAR.exe exists in directory \'utils\'\nfor Linux/Mac: need to install unrar (check README)') + print('\n[ERROR] Missing unrar tool\nfor Windows: make sure file UnRAR.exe exists in directory \'utils\'\nfor Linux/Mac: need to install unrar (check README)\n', flush=True) exit() def extract_7z(seven_zip_file: str, target_dir: str) -> None: @@ -73,4 +73,4 @@ def extract_file_to_dir(file_path: str, student_dir: str) -> None | Exception: elif file_path.lower().endswith('.7z'): extract_7z(file_path, student_dir) else: - print(f"\n[Error] unknown file type: {file_path}") + print(f"\n[ERROR] unknown file type: {file_path}\n", flush=True) diff --git a/utils/inspector.py b/utils/inspector.py index a8f4a41..13951a2 100644 --- a/utils/inspector.py +++ b/utils/inspector.py @@ -11,18 +11,18 @@ from utils.settings import CSV_DIR def load_excluded_filenames(submissions_dir_name: str) -> list[str]: # helper function for hashing all files csv_file_path = os.path.join(CSV_DIR, f'{submissions_dir_name}_excluded.csv') if not os.path.exists(csv_file_path): # if csv file with excluded file names for submission does not exist - print(f'[WARNING] Cannot find CSV file with list of excluded file names: {csv_file_path}\n[INFO] All files will be hashed & inspected') + print(f'[WARNING] Cannot find CSV file with list of excluded file names: {csv_file_path}\n[INFO] All files will be hashed & inspected', flush=True) return [] # return empty list to continue without any excluded file names else: # if csv file with excluded file names for submission exists try: df = pd.read_csv(csv_file_path) filename_list = df['exclude_filename'].tolist() # get the values of the 'filename' column as a list filename_list = [ f.lower() for f in filename_list ] # convert to lowercase for comparison with submission files - print(f'[INFO] Using CSV file with list of excluded file names: {csv_file_path}') + print(f'[INFO] Using CSV file with list of excluded file names: {csv_file_path}', flush=True) return filename_list except Exception as e: # any exception, print error and return empty list to continue without any excluded file names - print(f'[WARNING] Unable to load / read CSV file with list of excluded file names: {csv_file_path}\n[INFO] All files will be hashed & inspected') - print(f'[INFO] Error message: {e}') + print(f'[WARNING] Unable to load / read CSV file with list of excluded file names: {csv_file_path}\n[INFO] All files will be hashed & inspected', flush=True) + print(f'[INFO] Error message: {e}', flush=True) return [] @@ -61,7 +61,7 @@ def generate_hashes_gradebook(gradebook_dir_path: str) -> str: # main function writer = csv.DictWriter(csvfile, fieldnames=fieldnames) writer.writeheader() writer.writerows(dicts_with_hashes_list) - print(f'[INFO] Created CSV file with all files & hashes in gradebook: {gradebook_dir_name}\nCSV file: {csv_file_path}') + print(f'[INFO] Created CSV file with all files & hashes in gradebook: {gradebook_dir_name}\nCSV file: {csv_file_path}', flush=True) return csv_file_path @@ -95,7 +95,7 @@ def generate_hashes_submissions(submissions_dir_path: str) -> str: # main funct writer.writeheader() for student_dict in dicts_with_hashes_list: writer.writerows(student_dict) - print(f'[INFO] Created CSV file with all files & hashes for submissions in: {submissions_dir_name}\nCSV file: {csv_file_path}') + print(f'[INFO] Created CSV file with all files & hashes for submissions in: {submissions_dir_name}\nCSV file: {csv_file_path}', flush=True) return csv_file_path @@ -119,7 +119,7 @@ def generate_duplicate_hashes_generic(hashes_csv_file_path: str, drop_columns: l csv_out = hashes_csv_file_path.rsplit('_', 1)[0].replace('file_hashes', 'duplicate_') + datetime.now().strftime("%Y%m%d-%H%M%S") + '.csv' try: df_duplicate.to_csv(csv_out, index=False) - print(f'[INFO] Created CSV file with duplicate hashes in {gradebook_or_submissions_str}: {assignment_name}\nCSV file: {csv_out}') + print(f'[INFO] Created CSV file with duplicate hashes in {gradebook_or_submissions_str}: {assignment_name}\nCSV file: {csv_out}', flush=True) except Exception as e: exit(f'[ERROR] Something went wrong while trying to save csv file with duplicate hashes\nError message: {e}') diff --git a/utils/organiser.py b/utils/organiser.py index 7b19b11..121a5d1 100644 --- a/utils/organiser.py +++ b/utils/organiser.py @@ -2,7 +2,6 @@ import os, shutil, re from utils.extractor import extract_file_to_dir from utils.settings import BAD_DIR_NAME - def validate_gradebook_dir_name(src_dir: str) -> None: if not os.path.isdir(src_dir): # check if it exists and is a directory print(f"\n[Error] Incorrect directory: {src_dir}\n[Info] Make sure the directory exists in 'BB_gradebooks'") @@ -53,7 +52,7 @@ def get_gradebook_stats(src_dir: str) -> dict[str, int]: tracked_files_list = [ f'{files_counter[ext]} {ext}' for ext in tracked_file_extensions ] tracked_msg = f"{', '.join(str(f) for f in tracked_files_list)}" msg = f'\n[Stats] Gradebook contains {files_counter["all"]} file(s){dirs_msg}\n[Stats] Tracking {len(tracked_file_extensions)} file extension(s), files found: {tracked_msg}\n[Stats] Files with untracked extension: {files_counter["untracked"]}' - print(msg) + print(msg, flush=True) return files_counter @@ -88,10 +87,11 @@ def organise_gradebook(src_dir: str, dest_dir: str) -> None: """ validate_gradebook_dir_name(src_dir) # check if dir exists, and has files in it - exits if not os.makedirs(dest_dir, exist_ok=True) # create the destination directory if it doesn't exist - print('\nGetting gradebook stats...') + print('\nGetting gradebook stats...', flush=True) files_counter = get_gradebook_stats(src_dir) # print stats about the files in gradebook and get files_counter dict to use later students_numbers: list[str] = [] # list to add and count unique student numbers from all files in gradebook - print('\nStart organising...\n') + print('\nStart organising... (this may take a while depending on the number of submissions)\n', flush=True) + for file_name in os.listdir(src_dir): # iterate through all files in the directory if BAD_DIR_NAME not in file_name: # ignore dir BAD_DIR_NAME (created after first run if corrupt compressed files found) student_no = file_name.split('_attempt_')[0].split('_')[-1] # get student number from file name !! pattern might need adjusting if file name format from blackboard changes !! @@ -99,14 +99,14 @@ def organise_gradebook(src_dir: str, dest_dir: str) -> None: organise_file_per_student(src_dir, dest_dir, file_name, student_no) abs_path = os.getcwd() # absolute path of main script - print(f'[Info] Submissions organised into directory: {os.path.join(abs_path, dest_dir)}') - print(f'[Info] Unique student numbers in gradebook files: {len(set(students_numbers))}') + print(f'[Info] Submissions organised into directory: {os.path.join(abs_path, dest_dir)}', flush=True) + print(f'[Info] Unique student numbers in gradebook files: {len(set(students_numbers))}', flush=True) if files_counter['.txt'] == 0: - print(f'[Info] No submission text files found, file with comments not created') + print(f'[Info] No submission text files found, file with comments not created', flush=True) else: - print(f'[Info] Comments in file: {dest_dir}_comments.txt') + print(f'[Info] Comments in file: {dest_dir}_comments.txt', flush=True) - print(f'[Note] Compressed files (.zip, .rar, .7z) are automatically deleted from the gradebook directory after successful extraction') + print(f'[Note] Compressed files (.zip, .rar, .7z) are automatically deleted from the gradebook directory after successful extraction', flush=True) def check_submissions_dir_for_compressed(submissions_dir: str) -> None: