!pip install opendatasets --upgrade --quiet

import opendatasets as od

dataset_url = 'https://www.kaggle.com/datasets/usernam3/shopify-app-store'
od.download(dataset_url)

Please provide your Kaggle credentials to download this dataset. Learn more: http://bit.ly/kaggle-creds
Your Kaggle username:

import pandas as pd

df = pd.read_csv('./shopify-app-store/reviews.csv') \
        .groupby('rating') \
        .apply(lambda x: x.sample(n=2000, replace=True)) \
        .reset_index(drop=True)[['body', 'rating']] \
        .sample(frac=1)
df.head()

instruction = '''\
Classify the review as "helpful" or "not helpful". \
A "helpful" review should provide specific details about the user's experience, \
include clear suggestions or useful feedback, seem genuine and reflect a real user experience, \
be well-written and easy to understand, and directly pertain to the app's functionality.'''

labels = sorted(["helpful", "not helpful"])

!python -m sglang.launch_server --model-path meta-llama/Meta-Llama-3-8B-Instruct  --port 30000

!pip install "sglang[openai]"

from sglang import RuntimeEndpoint, function, gen, set_default_backend

HOST_URL = 'http://35.184.60.123:30000'

set_default_backend(RuntimeEndpoint(HOST_URL))

from transformers import AutoTokenizer

model_id = "meta-llama/Meta-Llama-3-8B-Instruct"
tokenizer = AutoTokenizer.from_pretrained(model_id)

@function
def run_sglang_gen(s, instruction, input_text, choices, temperature):
    messages = [{
        'role': 'user',
        'content': f'Instruction:\n\n{instruction}\n\nInput text:\n"""\n{input_text}\n"""\n'
    }]
    prompt = tokenizer.apply_chat_template(messages, add_generation_prompt=True, tokenize=False)
    s += prompt + gen(
        "answer",
        temperature=temperature,
        choices=choices,
    )

tokenizer_config.json:100%|██████████|51.0k/51.0k [00:00<00:00, 1.12MB/s]
tokenizer.json:100%|██████████|9.09M/9.09M [00:00<00:00, 43.6MB/s]
special_tokens_map.json:100%|██████████|73.0/73.0 [00:00<00:00, 2.09kB/s]
Special tokens have been added in the vocabulary, make sure the associated word embeddings are fine-tuned or trained.

import numpy as np
from tqdm import tqdm

batch_size = 10

def run_on_dataset(instruction, choices, texts):
    predictions = []
    for i in tqdm(range(0, len(texts), batch_size), total=len(texts) // batch_size):
        states = run_sglang_gen.run_batch([{
            'instruction': instruction,
            'input_text': t,
            'choices': choices,
            'temperature': 0.0
        } for t in texts[i:i+batch_size]])

        # collect outputs
        for state in states:
            meta_info = state.get_meta_info("answer")
            # convert log likelihoods to probabilities
            prob = np.exp(meta_info['normalized_prompt_logprob']) / np.sum(np.exp(meta_info['normalized_prompt_logprob']))
            predictions.append({
                'label': state['answer'],
                'prob': prob
            })
    return pd.DataFrame(predictions)

df[['label', 'prob']] = run_on_dataset(
    instruction=instruction,
    choices=labels,
    texts=df.body.tolist()
)

100%|██████████| 1000/1000 [03:30<00:00,  4.75it/s]

df['prob'] = df.prob.apply(max)

df['prob'].plot(kind='hist', bins=50)

df['issue_tp'] = df['prob'] < 0.51

print(f'Potential issues based on low token probability: {df["issue_tp"].sum()}')

Potential issues based on low token probability: 211

eval_instruction = '''Please act as an impartial judge and evaluate the response of the first language model. \
Assess if the model correctly classifies a product review based on the initial instruction as "helpful" or "not helpful". \
Check if the classification meets these criteria: specificity of user experience details, clarity and usefulness of feedback, \
authenticity and relevance to the app's functionality, and overall clarity and comprehension of the text. \
Label the response "Correct" if it fully meets all criteria, and "Incorrect" if it does not meet one or more of these criteria.'''

@function
def run_llm_eval(s, input_text, predicted_label):
    messages = [{
        'role': 'user',
        'content': (
            f'{eval_instruction}\n\n'
            f'Initial instruction:\n"""\n{instruction}\n"""\n\n'
            f'Product review:\n"""\n{input_text}\n"""\n\n'
            f'Classification result: {predicted_label}\n\n'
            'Evaluation result: '
        )
    }]
    prompt = tokenizer.apply_chat_template(messages, add_generation_prompt=True, tokenize=False)
    s += prompt + gen(
        "answer",
        temperature=0,
        choices=['Correct', 'Incorrect'],
    )


def run_eval_on_dataset(texts, labels):
    predictions = []
    texts_and_labels = list(zip(texts, labels))
    for i in tqdm(range(0, len(texts_and_labels), batch_size), total=len(texts) // batch_size):
        states = run_llm_eval.run_batch([{
            'input_text': t,
            'predicted_label': l,
        } for t, l in texts_and_labels[i:i+batch_size]])

        for state in states:
          # Identify errors
          predictions.append(True if state['answer'] == 'Incorrect' else False)

    return pd.DataFrame(predictions)


df['issue_lj'] = run_eval_on_dataset(df.body.tolist(), df.label.tolist())
print(f'Potential issues identified by cleanlab: {df["issue_lj"].sum()}')

100%|██████████| 1000/1000 [04:02<00:00,  4.12it/s]

# prepare the list of alternative instructions
instructions = [
    "Classify the review as \"helpful\" or \"not helpful\". A \"helpful\" review should offer specific details about the user's experience, contain clear suggestions or valuable feedback, appear genuine and reflect an authentic user experience, be well-composed and easy to comprehend, and directly relate to the app's functionality.",
    "Classify the review as \"helpful\" or \"not helpful\". A \"helpful\" review should present specific details about the user's experience, provide clear suggestions or beneficial feedback, seem authentic and mirror a real user experience, be well-crafted and straightforward to understand, and directly connect to the app's functionality.",
    "Classify the review as \"helpful\" or \"not helpful\". A \"helpful\" review should deliver specific details about the user's experience, include precise suggestions or advantageous feedback, appear genuine and reflect a true user experience, be well-written and simple to comprehend, and directly relate to the app's features."
]

# run each instruction through the same LLM
for i, instruction in enumerate(instructions):
  print(f'Running on instruction: "{instruction}"')
  df[[f'label_{i}', f'prob_{i}']] = run_on_dataset(
      instruction=instruction,
      choices=labels,
      texts=df.body.tolist()
  )

Running on instruction: "Classify the review as "helpful" or "not helpful". A "helpful" review should offer specific details about the user's experience, contain clear suggestions or valuable feedback, appear genuine and reflect an authentic user experience, be well-composed and easy to comprehend, and directly relate to the app's functionality."
100%|██████████| 1000/1000 [03:17<00:00,  5.07it/s]
Running on instruction: "Classify the review as "helpful" or "not helpful". A "helpful" review should present specific details about the user's experience, provide clear suggestions or beneficial feedback, seem authentic and mirror a real user experience, be well-crafted and straightforward to understand, and directly connect to the app's functionality."
100%|██████████| 1000/1000 [03:19<00:00,  5.02it/s]
Running on instruction: "Classify the review as "helpful" or "not helpful". A "helpful" review should deliver specific details about the user's experience, include precise suggestions or advantageous feedback, appear genuine and reflect a true user experience, be well-written and simple to comprehend, and directly relate to the app's features."
100%|██████████| 1000/1000 [03:22<00:00,  4.94it/s]

import seaborn as sns
import matplotlib.pyplot as plt
from collections import Counter

label_cols = [col for col in df.columns if col.startswith('label')]
label_to_num = {label: i for i, label in enumerate(sorted(labels))}
df['agreement'] = df[label_cols].apply(lambda x: max(Counter(x).values()) / len(x), axis=1)
corr = df[label_cols].replace(label_to_num).corr()
sns.heatmap(corr, annot=True, fmt=".2f", cmap='coolwarm')
plt.title('Label Correlation Heatmap')
plt.show()

from collections import Counter

df['issue_sc'] = df[label_cols].apply(lambda x: Counter(x).most_common(1)[0][0] != x['label'], axis=1)
print(f'Potential issues based on self consistency: {df["issue_sc"].sum()}')

Potential issues based on self consistency: 157

import os
from openai import OpenAI
from google.colab import userdata

os.environ['OPENAI_API_KEY'] = userdata.get('OPENAI_API_KEY')
client = OpenAI()

def get_embeddings(texts, model="text-embedding-3-small"):
    inputs = [str(text).replace("\n", " ") for text in texts]
    return [i.embedding for i in client.embeddings.create(input=inputs, model=model).data]

embs = []
for i in tqdm(range(0, len(df), 100)):
    embs.extend(get_embeddings(df.body.iloc[i:i + 100]))
embs = np.array(embs)
embs.shape

100%|██████████| 100/100 [01:15<00:00,  1.32it/s]
(10000, 1536)

from sklearn.linear_model import LogisticRegression
from sklearn.model_selection import cross_val_predict

model = LogisticRegression(max_iter=1000)

labels_num = np.array([label_to_num[label] for label in df.label])
pred_probs = cross_val_predict(estimator=model, X=embs, y=labels_num, cv=10, method="predict_proba")

!pip install cleanlab

from cleanlab.filter import find_label_issues

issue_idx = find_label_issues(labels_num, pred_probs, return_indices_ranked_by='self_confidence')
df['issue_cl'] = False
df.loc[issue_idx, 'issue_cl'] = True
print(f'Potential issues identified by cleanlab: {df["issue_cl"].sum()}')

Potential issues identified by cleanlab: 1556

cols_to_save = label_cols + ['body', 'issue_tp', 'issue_sc', 'issue_cl', 'issue_lj', 'prob', 'agreement']

df[cols_to_save].to_csv('finding_errors_in_llm_responses.csv', index=False)

# Install the package
$ pip install -U label-studio
# Launch Label Studio
$ label-studio