Update README.md

README.md

@@ -290,6 +290,44 @@ Ensure the input file exists, and specify the correct output path during decodin
 This design is flexible and reusable for various file types, making it a robust solution for encoding and decoding files into Base64.
 
 
+# Converting DataSets: 
+
+
+```python
+
+# Function to convert a PIL Image to a base64 string
+def image_to_base64(image):
+    buffered = io.BytesIO()
+    image.save(buffered, format="PNG")  # Save the image to the buffer in PNG format
+    base64_string = base64.b64encode(buffered.getvalue()).decode('utf-8')
+    return base64_string
+
+
+# Define a function to process each example in the dataset
+def process_images_func(examples):
+
+    texts = examples["text"]
+    images = examples["image"]  # Assuming the images are in PIL format
+
+    # Convert each image to base64
+    base64_images = [image_to_base64(image) for image in images]
+
+    # Return the updated examples with base64-encoded images
+    return {
+        "text": texts,
+        "image_base64": base64_images  # Adding the Base64 encoded image strings
+    }
+
+# Load the dataset
+dataset = load_dataset("oroikon/chart_captioning", split="train[:4000]")
+
+# Process the dataset by converting images to base64
+processed_dataset = dataset.map(process_images_func, batched=True)
+
+
+
+
+```
 # Prompt Engineering for Training:
 
 Early training involved embedding large, detailed prompts to improve the model’s depth of response and adaptability.
@@ -522,3 +560,52 @@ Keep the conversation going by always ending with a question to further probe th
 
 
 
+# ADDING EXTRA HEADS : 
+
+
+##  ADD HEAD
+
+
+# SPEECH-ENCODER-DECODER-MODEL
+```python
+
+
+print('Add Audio...')
+#Add Head
+# Combine pre-trained encoder and pre-trained decoder to form a Seq2Seq model
+_AudioFeatureExtractor = AutoFeatureExtractor.from_pretrained("openai/whisper-small")
+_AudioTokenizer = AutoTokenizer.from_pretrained("openai/whisper-small")
+_SpeechEncoderDecoder = SpeechEncoderDecoderModel.from_encoder_decoder_pretrained("openai/whisper-small","openai/whisper-small")
+
+# Add Pad tokems
+_SpeechEncoderDecoder.config.decoder_start_token_id = _AudioTokenizer.cls_token_id
+_SpeechEncoderDecoder.config.pad_token_id = _AudioTokenizer.pad_token_id
+LM_MODEL.SpeechEncoderDecoder = _SpeechEncoderDecoder
+# Add Sub Components
+LM_MODEL.Decoder_AudioTokenizer = _AudioTokenizer
+LM_MODEL.Encoder_AudioFeatureExtractor = _AudioFeatureExtractor
+LM_MODEL
+
+```
+
+
+# ADD HEAD
+# Combine pre-trained encoder and pre-trained decoder to form a Seq2Seq model
+
+```python
+
+Vmodel = VisionEncoderDecoderModel.from_encoder_decoder_pretrained(
+    "google/vit-base-patch16-224-in21k", "LeroyDyer/Mixtral_AI_Tiny"
+)
+_Encoder_ImageProcessor = Vmodel.encoder
+_Decoder_ImageTokenizer = Vmodel.decoder
+_VisionEncoderDecoderModel = Vmodel
+# Add Pad tokems
+LM_MODEL.VisionEncoderDecoder = _VisionEncoderDecoderModel
+# Add Sub Components
+LM_MODEL.Encoder_ImageProcessor = _Encoder_ImageProcessor
+LM_MODEL.Decoder_ImageTokenizer = _Decoder_ImageTokenizer
+LM_MODEL
+
+
+```