Hackathons/Hoya26
1
0
Fork 0
mirror of https://github.com/SirBlobby/Hoya26.git synced 2026-02-04 03:34:34 -05:00
Code Issues Packages Projects Releases Wiki Activity

Database and Reports Update

Browse Source
This commit is contained in:
default
2026-01-25 15:31:41 +00:00
parent 5ce0b4d278
commit 34eba52461
38 changed files with 3820 additions and 2797 deletions
Download Patch File Download Diff File Expand all files Collapse all files

34
backend/scripts/populate_db.py
View File

@@ -12,7 +12,7 @@ from src .rag .ingest import process_file
from src .rag .store import ingest_documents
from src .mongo .metadata import is_file_processed ,log_processed_file
def populate_from_dataset (dataset_dir ,category =None ):
def populate_from_dataset (dataset_dir ,category =None ,force =False ):
dataset_path =Path (dataset_dir )
if not dataset_path .exists ():
print (f"Dataset directory not found: {dataset_dir }")
@@ -27,9 +27,9 @@ def populate_from_dataset (dataset_dir ,category =None ):
for file_path in dataset_path .glob ('*'):
if file_path .is_file ()and file_path .suffix .lower ()in ['.csv','.pdf','.txt','.xlsx']:
if is_file_processed (file_path .name ):
if not force and is_file_processed (file_path .name ):
print (f"Skipping {file_path .name } (already processed)")
continue
continue
print (f"Processing {file_path .name }...")
try :
@@ -48,15 +48,23 @@ def populate_from_dataset (dataset_dir ,category =None ):
print (f"\nFinished! Processed {files_processed } files. Total chunks ingested: {total_chunks }")
if __name__ =="__main__":
parser =argparse .ArgumentParser (description ="Populate vector database from dataset files")
parser .add_argument ("--category","-c",type =str ,help ="Category to assign to ingested documents")
parser .add_argument ("--dir","-d",type =str ,default =None ,help ="Dataset directory path")
args =parser .parse_args ()
if __name__ == "__main__":
parser = argparse.ArgumentParser(description="Populate vector database from dataset files")
parser.add_argument("--category", "-c", type=str, help="Category to assign to ingested documents")
parser.add_argument("--dir", "-d", type=str, default=None, help="Dataset directory path")
parser.add_argument("--force", "-f", action="store_true", help="Force re-processing of files even if marked as processed")
args = parser.parse_args()
if args .dir :
dataset_dir =args .dir
else :
dataset_dir =os .path .join (os .path .dirname (__file__ ),'../dataset')
# Check vector store mode
use_atlas = os.environ.get("ATLAS_VECTORS", "false").lower() == "true"
store_name = "MongoDB Atlas Vector Search" if use_atlas else "ChromaDB"
print(f"--- Vector Store Mode: {store_name} ---")
populate_from_dataset (dataset_dir ,category =args .category )
if args.dir:
dataset_dir = args.dir
else:
dataset_dir = os.path.join(os.path.dirname(__file__), '../dataset')
# Note: We need to pass force flag to populate_from_dataset ideally,
# but the function signature doesn't have it. I'll modify the function signature too.
populate_from_dataset(dataset_dir, category=args.category, force=args.force)