03. Upload data set for evaluation generated (HuggingFace Dataset)

from dotenv import load_dotenv

load_dotenv()

 True 

# Set up LangSmith tracking. https://smith.langchain.com
# !pip install -qU langchain-teddynote
from langchain_teddynote import logging

# Enter a project name.
logging.langsmith("CH16-Evaluations")

 Start tracking LangSmith. 
[Project name] 
CH16-Evaluations 

import pandas as pd

df = pd.read_csv("data/ragas_synthetic_dataset.csv")
df.head()

Translation using DeepL

• DeepL API key issuance

Key issued .env To file DEEPL_API_KEY Save as.

# !pip install -qU deepl

import os
from langchain_teddynote.translate import Translator

# api key setting
deepl_api_key = os.getenv("DEEPL_API_KEY")

# Object creation
translator = Translator(deepl_api_key, "EN", "KO")

# Run translation
translated_text = translator("hello, nice to meet you")
print(translated_text)

 Hello, nice to meet you

from tqdm import tqdm

# 번역
for i, row in tqdm(df.iterrows(), total=len(df), desc="번역 진행 중"):
    df.loc[i, "question_translated"] = translator(row["question"])
    df.loc[i, "ground_truth_translated"] = translator(row["ground_truth"])

 Translation in progress: 100%|██████████| 10/10 [00:09<00:00, 1.05it/s]

Before translation

# question, ground_truth Check the fever.
df.loc[:, ["question", "ground_truth"]].head()

Copy# question_translated, ground_truth_translated 열을 확인합니다.
df.loc[:, ["question_translated", "ground_truth_translated"]].head()

Check the translated results.

# question, ground_truth 열을 삭제하고 question_translated, ground_truth_translated 열의 이름을 변경합니다.
df.drop(columns=["question", "ground_truth"], inplace=True)
df.rename(
    columns={
        "question_translated": "question",
        "ground_truth_translated": "ground_truth",
    },
    inplace=True,
)
df.head()

# Save the translated dataset.
df.to_csv("data/ragas_synthetic_dataset_translated.csv", index=False)

Hugging Face Dataset Upload

Convert Pandas DataFrame to Hugging Face Dataset to proceed with upload.

from datasets import Dataset

# pandas DataFrame을 Hugging Face Dataset으로 변환
dataset = Dataset.from_pandas(df)

# Check the dataset
print(dataset)

 Dataset ({ 
    features: ['contexts','evolution_type','metadata','episode_done','question','ground_truth'], 
    num_rows: 10 
}) 

from datasets import Dataset
import os

# pandas DataFrame을 Hugging Face Dataset으로 변환
dataset = Dataset.from_pandas(df)

# Set dataset name (change to whatever name you want)
dataset_name = "teddylee777/rag-synthetic-dataset"

# Upload dataset
dataset.push_to_hub(
    dataset_name,
    private=True,  # private=False로 설정하면 공개 데이터셋이 됩니다.
    split="korean_v1",  # 데이터셋 split 이름 입력
    token=os.getenv("HUGGINGFACEHUB_API_TOKEN"),
)

[Note] Dataset Viewer may take some time to be written.