Update process_hf_dataset.py

process_hf_dataset.py

@@ -186,11 +186,11 @@ def process_hf_dataset(batch_size=100, use_gpu=False):
     # Do not clear or populate with defaults here—let UI buttons handle this
     try:
         collection = client.get_or_create_collection(DB_NAME)
-        logger.info(f"Using existing or new ChromaDB collection: {DB_NAME}")
+        logger.info(f"Using existing or new ChromaDB collection: {DB_NAME}, contains {collection.count()} entries")
         # Verify collection is valid
         if collection is None or not hasattr(collection, 'add'):
             raise ValueError("ChromaDB collection access failed")
-        logger.info(f"ChromaDB collection verified, contains {collection.count()} entries")
+        logger.info("Verified ChromaDB collection is valid")
     except Exception as e:
         logger.error(f"Error accessing ChromaDB collection: {e}")
         raise
@@ -277,8 +277,8 @@ def save_chromadb_to_hf(dataset_name=HF_DATASET_NAME, token=os.getenv("HF_KEY"))
         logger.info(f"Created Hugging Face Dataset with {len(data['code'])} entries")
         
         # Push to Hugging Face Hub
-        dataset.push_to_hub(dataset_name, token=token)
-        logger.info(f"Dataset pushed to Hugging Face Hub as {dataset_name}")
+        dataset.push_to_hub(dataset_name, token=token, exist_ok=True)  # Allow overwriting existing dataset
+        logger.info(f"Dataset pushed to Hugging Face Hub as {dataset_name}, overwriting existing dataset")
         # Verify push (optional, could check dataset on Hub)
         logger.info(f"Verified Hugging Face dataset push with {len(dataset)} entries")
     except Exception as e: