initial commit

.gitattributes

@@ -33,3 +33,4 @@ saved_model/**/* filter=lfs diff=lfs merge=lfs -text
 *.zip filter=lfs diff=lfs merge=lfs -text
 *.zst filter=lfs diff=lfs merge=lfs -text
 *tfevents* filter=lfs diff=lfs merge=lfs -text
+*.pdf filter=lfs diff=lfs merge=lfs -text

LICENSE

@@ -0,0 +1,21 @@
+MIT License
+
+Copyright (c) 2023 AI Anytime
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.

chainlit.md

@@ -0,0 +1,4 @@
+# Welcome to Llama2 Med-Bot! 🚀🤖
+
+Hi there, 👋 We're excited to have you on board. This is a powerful bot designed to help you ask queries related to your data/knowledge.
+

data/geo-10-3-notes.pdf

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:5e627281b6bad3fc951001e4740c976e5916255758c87d6d2eea0f5f5c7adf29
+size 220267

data/geo-10/geo-10-1.pdf

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:3f869020fed2bd143a8044316fd9071c9b194714ca3b19da19b7f80a577f77c3
+size 228700

data/geo-10/geo-chapter-list.json

@@ -0,0 +1,64 @@
+[
+    {
+        "chapter_number": 1,
+        "chapter_name": "Resources and Development",
+        "subtopics": [
+            "Definition of Resource",
+            "Transformation Process in the Environment",
+            "Classification of Resources",
+            "Development of Resources",
+            "Problems Due to Indiscriminate Resource Use",
+            "Importance of Equitable Resource Distribution",
+            "Resource Planning",
+            "Sustainable Development",
+            "Rio de Janeiro Earth Summit, 1992",
+            "Agenda 21",
+            "Resource Planning in India",
+            "Factors in Resource Planning",
+            "Role of Technology and Institutions in Resource Development",
+            "Conservation of Resources",
+            "Historical Perspectives on Resource Conservation",
+            "Land Resources Importance",
+            "Land Utilization Types",
+            "Land Use Pattern in India",
+            "Forest Area Concerns",
+            "Land Degradation and Conservation Measures",
+            "Soil as a Resource and Its Formation",
+            "Classification of Soils",
+            "Alluvial Soils",
+            "Black Soil",
+            "Red and Yellow Soils",
+            "Laterite Soil",
+            "Arid Soils",
+            "Forest Soils",
+            "Soil Erosion and Conservation Methods"
+        ]
+    },
+    {
+      "chapter_number": 2,
+      "chapter_name": "In-Depth Analysis",
+      "subtopics": [
+        "Detailed Aspect 1",
+        "Detailed Aspect 2",
+        "Case Studies on the Topic"
+      ]
+    },
+    {
+      "chapter_number": 3,
+      "chapter_name": "Advanced Applications",
+      "subtopics": [
+        "Advanced Concept 1",
+        "Advanced Concept 2",
+        "Future Trends in the Field"
+      ]
+    },
+    {
+      "chapter_number": 4,
+      "chapter_name": "Conclusion and Summary",
+      "subtopics": [
+        "Recap of Key Points",
+        "Final Thoughts",
+        "Further Resources"
+      ]
+    }
+]

data/jess1ps_merged.pdf

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:057d8de0e37b5acc1975629794a147b54d42851f43d5e4cb3c9dec68c6428513
+size 16202519

index.pkl

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:cdce5bc50acb24703cf60f20d7b36783c00758713025be3bf803cc7430127d61
+size 252285

ingest.py

@@ -0,0 +1,30 @@
+from langchain.embeddings import HuggingFaceEmbeddings
+from langchain.vectorstores import FAISS
+from langchain.document_loaders import PyPDFLoader, DirectoryLoader
+from langchain.text_splitter import RecursiveCharacterTextSplitter, SpacyTextSplitter
+
+DATA_PATH = 'data/geo-10'
+DB_FAISS_PATH = 'refbooks-vectorstore/geo-10-1'
+
+# Create vector database
+
+
+def create_vector_db():
+    loader = DirectoryLoader(DATA_PATH,
+                             glob='geo-10-1.pdf',
+                             loader_cls=PyPDFLoader)
+
+    documents = loader.load()
+    text_splitter = SpacyTextSplitter(chunk_size=500,
+                                                   chunk_overlap=50)
+    texts = text_splitter.split_documents(documents)
+
+    embeddings = HuggingFaceEmbeddings(model_name='sentence-transformers/all-MiniLM-L6-v2',
+                                       model_kwargs={'device': 'cpu'})
+
+    db = FAISS.from_documents(texts, embeddings)
+    db.save_local(DB_FAISS_PATH)
+
+
+if __name__ == "__main__":
+    create_vector_db()

learnai.py

@@ -0,0 +1,148 @@
+import streamlit as st
+import random
+import time
+import json
+
+from llm_loader import get_llm_response
+
+from langchain.prompts import PromptTemplate
+
+age = 15
+name = 'Fidva'
+grade = 10
+context = '{context}'
+question = '{question}'
+
+DB_FAISS_PATH = 'refbooks-vectorstore/geo-10-1'
+JSON_FILE_PATH = 'data/geo-10/geo-chapter-list.json'
+
+
+with open(JSON_FILE_PATH, 'r') as file:
+    book_data = json.load(file)
+    
+chapter_data = book_data[0]
+subtopics = chapter_data['subtopics']
+
+######## Streamlit App ########
+
+st.title("Learn.AI")
+
+# Add custom CSS properties
+st.markdown(
+    """
+    <style>
+    @import url('https://fonts.googleapis.com/css2?family=Sometype+Mono:wght@400;500;600;700&display=swap');
+    
+    :root {
+        --color-bg: #03045E;
+        --color-footer: #0077B6;
+        --color-component: #00B4D8;
+        --font-family: 'Sometype Mono', sans-serif;
+        --font-color: #FAFAFA;
+    }
+    body, .stTextArea textarea {
+        font-family: var(--font-family); !important;
+        background-color: var(--color-bg);
+        color: var(--font-color); !important;
+    }
+    h1, p, li {
+        font-family: var(--font-family);
+        color: var(--font-color);
+    }
+    textarea {
+        font-family: var(--font-family);
+        color: var(--font-color);
+        background-color: #FAFAFA;
+    }
+    .stApp {
+        background-color: var(--color-bg);
+    }
+    .stChatMessage:nth-child(even) .stMarkdown {
+        background-color: var(--color-bg);
+    }
+    .stChatFloatingInputContainer {
+        background-color: var(--color-bg);
+        color: var(--font-color);
+    }
+    .st-emotion-cache-1avcm0n ezrtsby2 {
+        background-color: #050647;
+    }
+    .st-emotion-cache-10trblm {
+        --font-family: 'Sometype Mono', sans-serif;
+    }
+    .st-emotion-cache-nahz7x {
+        --font-family: 'Sometype Mono', sans-serif;
+    }
+    </style>
+    """,
+    unsafe_allow_html=True
+)
+
+
+# Initialize chat history
+if "messages" not in st.session_state:
+    st.session_state.messages = []
+
+# Display chat messages from history on app rerun
+for message in st.session_state.messages:
+    with st.chat_message(message["role"]):
+        st.markdown(message["content"])
+
+if "lesson_count" not in st.session_state:
+    st.session_state.lesson_count = 0
+# GreetingMessage = f'Hi there, {name}! Let\'s start the lesson! Type \'start\' when you\'re ready to begin!'
+# st.session_state.messages.append({"role": "assistant", "content": GreetingMessage})
+
+# Accept user input
+if prompt := st.chat_input(f'Hi there, {name}! Let\'s start the lesson! Type \'start\' when you\'re ready to begin!'):
+    print("TOPIC NAME:",subtopics[st.session_state.lesson_count])
+    print("USER:",prompt)
+    if prompt.lower() =='start' or prompt.lower()=='continue' or prompt.lower()=='next':
+        print("Topic Name:", subtopics[st.session_state.lesson_count])
+        # Display assistant response in chat message container
+        with st.chat_message("assistant"):
+            placeholder_list = ['Writing notes...', 'Revising topic...', 'Clearing blackboard...', 'Formulating Lesson Plan...', 'Getting ready for doubts...']
+            placeholder_text = random.choice(placeholder_list)
+            with st.spinner(placeholder_text):
+                assistant_response = get_llm_response(subtopics[st.session_state.lesson_count], template_type='lesson')
+                st.session_state.lesson_count += 1
+                message_placeholder = st.empty()
+                full_response = ""
+                
+                # Simulate stream of response with milliseconds delay\
+                print(assistant_response)
+                for chunk in assistant_response.split():
+                    full_response += chunk + " "
+                    time.sleep(0.05)
+                    # Add a blinking cursor to simulate typing
+                    message_placeholder.markdown(full_response + "▌")
+                message_placeholder.markdown(assistant_response)
+            
+        st.session_state.messages.append({"role": "assistant", "content": assistant_response})
+        
+    else:
+        # Add user message to chat history
+        st.session_state.messages.append({"role": "user", "content": prompt})
+        # Display user message in chat message container
+        with st.chat_message("user"):
+            st.markdown(prompt)
+
+        # Display assistant response in chat message container
+        with st.chat_message("assistant"):
+            placeholder_list = ['Thinking...', 'Reading Textbook...', 'Clearing blackboard...', 'Revising Topics...', 'Refilling pen...']
+            placeholder_text = random.choice(placeholder_list)
+            with st.spinner(placeholder_text):
+                assistant_response = get_llm_response(prompt, template_type='user')
+                message_placeholder = st.empty()
+                full_response = ""
+                
+                # Simulate stream of response with milliseconds delay\
+                print(assistant_response)
+                for chunk in assistant_response.split():
+                    full_response += chunk + " "
+                    time.sleep(0.05)
+                    # Add a blinking cursor to simulate typing
+                    message_placeholder.markdown(full_response + "▌")
+                message_placeholder.markdown(assistant_response)
+        # Add assistant response to chat history
+        st.session_state.messages.append({"role": "assistant", "content": assistant_response})

learnai_chainlit.py

@@ -0,0 +1,25 @@
+import json
+import chainlit as cl
+
+from llm_loader import get_llm_response
+
+from langchain.prompts import PromptTemplate
+
+
+age = 15
+name = "Fidva"
+grade = 10
+context = '{context}'
+question = '{question}'
+
+DB_FAISS_PATH = 'refbooks-vectorstore/geo-10-1'
+JSON_FILE_PATH = 'data/geo-10/geo-chapter-list.json'
+
+# Load book data
+with open(JSON_FILE_PATH, 'r') as file:
+    book_data = json.load(file)
+    
+chapter_data = book_data[0]
+subtopics = chapter_data['subtopics']
+
+# Chainlit App

llm_loader.py

@@ -0,0 +1,104 @@
+from langchain.embeddings import HuggingFaceEmbeddings
+from langchain.vectorstores import FAISS
+from langchain.llms import CTransformers
+from langchain.chains import RetrievalQA
+from langchain.prompts import PromptTemplate
+from langchain.callbacks.streaming_stdout import StreamingStdOutCallbackHandler
+
+name = 'Fidva'
+age = 15
+grade = 10
+context = '{context}'
+question = '{question}'
+
+DB_FAISS_PATH = 'refbooks-vectorstore/geo-10-1'
+
+
+def get_llm_response(query, template_type):
+
+    if template_type == 'user':
+        user_template = """Use the following pieces of information to answer the user's question in a friendly way.
+        If you don't know the answer, just say that you don't know, don't try to make up an answer. Also refer to the user by their name, and keep in mind their age while answering the question.
+
+        Name of user: {name}
+        Age of user: {age}
+        Grade of user: {grade}
+        Context: {context}
+        Question: {question}
+
+        Return the Helpful Answer, and then also give the user a Knowledge Check Question related to what he just asked.
+        Returning the helpful answer is a must and takes higher priority.
+
+        Helpful answer:
+        """
+        unformatted_prompt_template = PromptTemplate.from_template(
+            user_template)
+
+    elif template_type == 'lesson':
+        # lesson_template = """Teach the given topic in accordance with the content below to the user in a friendly way, while keeping in mind the user's age and his grade.
+        # Name of user: {name}
+        # Age of user: {age}
+        # Grade of user: {grade}
+        # Content: {context}
+        # Topic: {question}
+        # """
+        lesson_template = """Hello {name}! Let's dive into the topic of {question} together.
+    
+As a {grade}th grader at {age} years old, it's great to explore this subject!
+
+Let's start by understanding the context:
+    
+    {context}
+
+Now, to grasp this topic better, here are some key points to consider:
+    
+    - Explain the fundamental concept or idea related to {question}.
+    
+    - Provide examples or illustrations to make it easier to comprehend.
+    
+    - Share any real-life applications or relevance of this topic.
+
+Feel free to ask if you have any questions along the way. Let's learn together!
+"""
+        unformatted_prompt_template = PromptTemplate.from_template(
+            lesson_template)
+
+    prompt_template = unformatted_prompt_template.format(
+        name=name,
+        age=age,
+        grade=grade,
+        context=context,
+        question=question
+    )
+
+    embeddings = HuggingFaceEmbeddings(model_name="sentence-transformers/all-MiniLM-L6-v2",
+                                       model_kwargs={'device': 'cpu'})
+    db = FAISS.load_local(DB_FAISS_PATH, embeddings)
+
+    # Load the locally downloaded model here
+    llm = CTransformers(
+        model="TheBloke/Llama-2-7B-Chat-GGML",
+        model_type="llama",
+        callbacks=[StreamingStdOutCallbackHandler()],
+        config={
+            'context_length': 4096,
+            'temperature': 0.1,
+            'max_new_tokens': 512,
+        },
+    )
+
+    qa_prompt = PromptTemplate(template=prompt_template,
+                               input_variables=['context', 'question'])
+
+    print(qa_prompt)
+    qa_result = RetrievalQA.from_chain_type(llm=llm,
+                                            chain_type='stuff',
+                                            retriever=db.as_retriever(
+                                                search_kwargs={'k': 1}),
+                                            return_source_documents=True,
+                                            chain_type_kwargs={
+                                                'prompt': qa_prompt},
+                                            )
+
+    response = qa_result({'query': query})
+    return response['result']

model.py

@@ -0,0 +1,140 @@
+from langchain.embeddings import HuggingFaceEmbeddings
+from langchain.vectorstores import FAISS
+from langchain.llms import CTransformers
+from langchain.chains import RetrievalQA
+from langchain.prompts import PromptTemplate
+
+import chainlit as cl
+
+age = 15
+name = 'Fidva'
+grade = 10
+context = '{context}'
+question = '{question}'
+
+DB_FAISS_PATH = 'vectorstore/db_faiss/geo-10-whole_book'
+# OPENAI_API_KEY = "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"
+
+custom_prompt_template = """Use the following pieces of information to answer the user's question in a friendly way.
+If you don't know the answer, just say that you don't know, don't try to make up an answer. Also refer to the user by their name, and keep in mind their age while answering the question.
+
+Name of user: {name}
+Age of user: {age}
+Grade of user: {grade}
+Context: {context}
+Question: {question}
+
+Return the Helpful Answer, and then also give the user a Knowledge Check Question related to what he just asked.
+Returning the helpful answer is a must and takes higher priority.
+
+Helpful answer:
+"""
+
+custom_prompt_template = PromptTemplate.from_template(custom_prompt_template)
+formatted_prompt = custom_prompt_template.format(
+    name=name, age=age, grade=grade, context=context, question=question)
+
+
+def set_custom_prompt():
+    """
+    Prompt template for QA retrieval for each vectorstore
+    """
+    prompt = PromptTemplate(template=formatted_prompt,
+                            input_variables=['context', 'question'])
+    return prompt
+
+# Retrieval QA Chain
+
+
+def retrieval_qa_chain(llm, prompt, db):
+    qa_chain = RetrievalQA.from_chain_type(llm=llm,
+                                           chain_type='stuff',
+                                           retriever=db.as_retriever(
+                                               search_kwargs={'k': 1}),
+                                           return_source_documents=True,
+                                           chain_type_kwargs={'prompt': prompt}
+                                           )
+    return qa_chain
+
+# Loading the model
+
+
+def load_llm():
+
+    # config = AutoConfig.from_pretrained("TheBloke/Llama-2-7B-Chat-GGML")
+    # config.max_seq_len = 4096
+    # config.max_answer_len = 1024
+
+    # Load the locally downloaded model here
+    llm = CTransformers(
+        model="TheBloke/Llama-2-7B-Chat-GGML",
+        # model = "zephyr-7b-beta.Q5_K_S.gguf",
+        model_type="llama",
+        config={
+            'context_length': 4096,
+            'temperature': 0.3,
+            'max_new_tokens': 512,
+        },
+    )
+
+    # llm = AutoModelForCausalLM.from_pretrained(
+    #     "TheBloke/Llama-2-7B-Chat-GGML",
+    #     model_type="llama",
+    #     config=config,
+    #     temperature=0.5
+    # )
+
+    return llm
+
+# QA Model Function
+
+
+def qa_bot():
+    embeddings = HuggingFaceEmbeddings(model_name="sentence-transformers/all-MiniLM-L6-v2",
+                                       model_kwargs={'device': 'cpu'})
+    db = FAISS.load_local(DB_FAISS_PATH, embeddings)
+    llm = load_llm()
+    qa_prompt = set_custom_prompt()
+    qa = retrieval_qa_chain(llm, qa_prompt, db)
+
+    return qa
+
+# output function
+
+
+def final_result(query):
+    qa_result = qa_bot()
+    response = qa_result({'query': query})
+    return response
+
+# chainlit code
+
+
+@cl.on_chat_start
+async def start():
+    chain = qa_bot()
+    msg = cl.Message(content="Starting the bot...")
+    await msg.send()
+    msg.content = "Hi, Welcome to Geo Bot. What is your query?"
+    await msg.update()
+
+    cl.user_session.set("chain", chain)
+
+
+@cl.on_message
+async def main(message: cl.Message):
+    chain = cl.user_session.get("chain")
+    cb = cl.AsyncLangchainCallbackHandler(
+        stream_final_answer=True, answer_prefix_tokens=["FINAL", "ANSWER"]
+    )
+    cb.answer_reached = True
+    res = await chain.acall(message.content, callbacks=[cb])
+    answer = res["result"]
+    sources = res["source_documents"]
+
+    if sources:
+        answer += f"\nSources:" + str(sources)
+    else:
+        answer += "\nNo sources found"
+
+    await cl.Message(content=answer).send()

refbooks-vectorstore/geo-10-1/index.pkl

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:139f9355d0fdc3c31b2f4f35fe7de6bf92f01af1349dfefb381571b4aee6a836
+size 29329

requirements.txt

@@ -0,0 +1,13 @@
+accelerate==0.25.0
+bitsandbytes==0.41.2.post2
+chainlit==0.7.700
+ctransformers==0.2.27
+exceptiongroup==1.2.0
+faiss-cpu==1.7.4
+langchain==0.0.345
+openai==1.3.7
+pickleshare==0.7.5
+pip-chill==1.0.3
+pypdf==3.17.1
+sentence-transformers==2.2.2
+streamlit==1.29.0

testing.ipynb

@@ -0,0 +1,68 @@
+{
+ "cells": [
+  {
+   "cell_type": "code",
+   "execution_count": 3,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "import json"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 15,
+   "metadata": {},
+   "outputs": [
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "<class 'list'>\n",
+      "<class 'dict'>\n",
+      "{'chapter_number': 1, 'chapter_name': 'Resources and Development', 'subtopics': ['Definition of Resource', 'Transformation Process in the Environment', 'Classification of Resources', 'Development of Resources', 'Problems Due to Indiscriminate Resource Use', 'Importance of Equitable Resource Distribution', 'Resource Planning', 'Sustainable Development', 'Rio de Janeiro Earth Summit, 1992', 'Agenda 21', 'Resource Planning in India', 'Factors in Resource Planning', 'Role of Technology and Institutions in Resource Development', 'Conservation of Resources', 'Historical Perspectives on Resource Conservation', 'Land Resources Importance', 'Land Utilization Types', 'Land Use Pattern in India', 'Forest Area Concerns', 'Land Degradation and Conservation Measures', 'Soil as a Resource and Its Formation', 'Classification of Soils', 'Alluvial Soils', 'Black Soil', 'Red and Yellow Soils', 'Laterite Soil', 'Arid Soils', 'Forest Soils', 'Soil Erosion and Conservation Methods']}\n",
+      "['Definition of Resource', 'Transformation Process in the Environment', 'Classification of Resources', 'Development of Resources', 'Problems Due to Indiscriminate Resource Use', 'Importance of Equitable Resource Distribution', 'Resource Planning', 'Sustainable Development', 'Rio de Janeiro Earth Summit, 1992', 'Agenda 21', 'Resource Planning in India', 'Factors in Resource Planning', 'Role of Technology and Institutions in Resource Development', 'Conservation of Resources', 'Historical Perspectives on Resource Conservation', 'Land Resources Importance', 'Land Utilization Types', 'Land Use Pattern in India', 'Forest Area Concerns', 'Land Degradation and Conservation Measures', 'Soil as a Resource and Its Formation', 'Classification of Soils', 'Alluvial Soils', 'Black Soil', 'Red and Yellow Soils', 'Laterite Soil', 'Arid Soils', 'Forest Soils', 'Soil Erosion and Conservation Methods']\n"
+     ]
+    }
+   ],
+   "source": [
+    "DB_FAISS_PATH = 'refbooks-vectorstore/geo-10-1'\n",
+    "JSON_FILE_PATH = 'data/geo-10/geo-chapter-list.json'\n",
+    "\n",
+    "with open(JSON_FILE_PATH, 'r') as file:\n",
+    "    book_data = json.load(file)\n",
+    "    print(type(book_data))\n",
+    "    \n",
+    "chapter_data = book_data[0]\n",
+    "print(type(chapter_data))\n",
+    "print(chapter_data)\n",
+    "\n",
+    "chapter_name = chapter_data['chapter_name']\n",
+    "subtopics = chapter_data['subtopics']\n",
+    "\n",
+    "print(subtopics)"
+   ]
+  }
+ ],
+ "metadata": {
+  "kernelspec": {
+   "display_name": "Python 3",
+   "language": "python",
+   "name": "python3"
+  },
+  "language_info": {
+   "codemirror_mode": {
+    "name": "ipython",
+    "version": 3
+   },
+   "file_extension": ".py",
+   "mimetype": "text/x-python",
+   "name": "python",
+   "nbconvert_exporter": "python",
+   "pygments_lexer": "ipython3",
+   "version": "3.11.5"
+  }
+ },
+ "nbformat": 4,
+ "nbformat_minor": 2
+}

vectorstore/db_faiss/geo-10-3/index.pkl

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:7c6a4d112990968a29d9e16897bd5b5a166acec3a894df2daf10419a310c6d72
+size 7312

vectorstore/db_faiss/geo-10-whole_book/index.pkl

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:b03d45e671070a77668a6d3431509d26e5d792549418c2070a6b4b491e3fe597
+size 252285

vectorstore/db_faiss/index.pkl

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:9fc76b583cb25e343ab9bc3292f3096035eba770f775d73246902c1151d9a54b
+size 252285

word_docs/testing.ipynb

@@ -0,0 +1,68 @@
+{
+ "cells": [
+  {
+   "cell_type": "code",
+   "execution_count": 3,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "import json"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 15,
+   "metadata": {},
+   "outputs": [
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "<class 'list'>\n",
+      "<class 'dict'>\n",
+      "{'chapter_number': 1, 'chapter_name': 'Resources and Development', 'subtopics': ['Definition of Resource', 'Transformation Process in the Environment', 'Classification of Resources', 'Development of Resources', 'Problems Due to Indiscriminate Resource Use', 'Importance of Equitable Resource Distribution', 'Resource Planning', 'Sustainable Development', 'Rio de Janeiro Earth Summit, 1992', 'Agenda 21', 'Resource Planning in India', 'Factors in Resource Planning', 'Role of Technology and Institutions in Resource Development', 'Conservation of Resources', 'Historical Perspectives on Resource Conservation', 'Land Resources Importance', 'Land Utilization Types', 'Land Use Pattern in India', 'Forest Area Concerns', 'Land Degradation and Conservation Measures', 'Soil as a Resource and Its Formation', 'Classification of Soils', 'Alluvial Soils', 'Black Soil', 'Red and Yellow Soils', 'Laterite Soil', 'Arid Soils', 'Forest Soils', 'Soil Erosion and Conservation Methods']}\n",
+      "['Definition of Resource', 'Transformation Process in the Environment', 'Classification of Resources', 'Development of Resources', 'Problems Due to Indiscriminate Resource Use', 'Importance of Equitable Resource Distribution', 'Resource Planning', 'Sustainable Development', 'Rio de Janeiro Earth Summit, 1992', 'Agenda 21', 'Resource Planning in India', 'Factors in Resource Planning', 'Role of Technology and Institutions in Resource Development', 'Conservation of Resources', 'Historical Perspectives on Resource Conservation', 'Land Resources Importance', 'Land Utilization Types', 'Land Use Pattern in India', 'Forest Area Concerns', 'Land Degradation and Conservation Measures', 'Soil as a Resource and Its Formation', 'Classification of Soils', 'Alluvial Soils', 'Black Soil', 'Red and Yellow Soils', 'Laterite Soil', 'Arid Soils', 'Forest Soils', 'Soil Erosion and Conservation Methods']\n"
+     ]
+    }
+   ],
+   "source": [
+    "DB_FAISS_PATH = 'refbooks-vectorstore/geo-10-1'\n",
+    "JSON_FILE_PATH = 'data/geo-10/geo-chapter-list.json'\n",
+    "\n",
+    "with open(JSON_FILE_PATH, 'r') as file:\n",
+    "    book_data = json.load(file)\n",
+    "    print(type(book_data))\n",
+    "    \n",
+    "chapter_data = book_data[0]\n",
+    "print(type(chapter_data))\n",
+    "print(chapter_data)\n",
+    "\n",
+    "chapter_name = chapter_data['chapter_name']\n",
+    "subtopics = chapter_data['subtopics']\n",
+    "\n",
+    "print(subtopics)"
+   ]
+  }
+ ],
+ "metadata": {
+  "kernelspec": {
+   "display_name": "Python 3",
+   "language": "python",
+   "name": "python3"
+  },
+  "language_info": {
+   "codemirror_mode": {
+    "name": "ipython",
+    "version": 3
+   },
+   "file_extension": ".py",
+   "mimetype": "text/x-python",
+   "name": "python",
+   "nbconvert_exporter": "python",
+   "pygments_lexer": "ipython3",
+   "version": "3.11.5"
+  }
+ },
+ "nbformat": 4,
+ "nbformat_minor": 2
+}