def turn_into_features(df, models):
    '''
    Convert pairwise battle results into feature matrix X and label vector y
    suitable for logistic regression based on the Bradley-Terry model
    '''
    # TODO:
    # 1. Iterate through each row in the DataFrame.
    # 2. For each battle, create a feature vector:
    #    - Assign +1 to the column corresponding to 'model_a'.
    #    - Assign -1 to the column corresponding to 'model_b'.
    #    - All other columns should be 0.
    # 3. Append the label:
    #    - 1 if 'model_a' is the winner.
    #    - 0 if 'model_b' is the winner.
    # 4. Return the feature matrix X and label vector y as numpy arrays.
    X = []
    y = []


    for _ ,row in df.iterrows():
        a=row['model_a']
        b=row['model_b']

        win_a=1 if row['winner']==a else 0

        x_ab=[1 if col==a else -1 if col==b else 0 for col in models]

        y_ab=win_a

        x_ba=[1 if col==b else -1 if col==a else 0  for col in models]

        y_ba=1-win_a

        X.append(x_ab)
        y.append(y_ab)
        X.append(x_ba)
        y.append(y_ba)
    return np.array(X), np.array(y)

X, y = turn_into_features(selected_battles_no_ties, selected_models)
X.shape, y.shape

def get_bootstrapped_score(X, y, models, category_name="Overall", n_bootstrap=10):
    """
    Bootstraps logistic regression model scores to estimate confidence intervals.
    Args:
        X: Feature matrix
        y: Labels
        models: List of model names (order matches columns of X)
        n_bootstrap: Number of bootstrap samples
    Returns:
        results_df: DataFrame with Model, Average Score, Lower Bound, Upper Bound
        mean_scores: Mean of bootstrapped scores (np.array)
        confidence_intervals: 2.5 and 97.5 percentiles (np.array shape [2, n_models])
    """
    #TODO

    np.random.seed(189)  # for reproducibility
    bootstrap_scores = []
    for i in range(n_bootstrap):
        indices = np.random.choice(len(X), size=len(X), replace=True)
        X=X[indices]
        y=y[indices]
        model = LogisticRegression(fit_intercept=False)
        model.fit(X,y)
        bootstrap_scores.append(model.coef_[0])
    bootstrap_scores = np.array(bootstrap_scores)
    mean_scores = bootstrap_scores.mean(axis=0)
    confidence_intervals = np.percentile(bootstrap_scores, [2.5, 97.5], axis=0)

    results = {
        "Model": models,
        "Average Score": mean_scores,
        "Lower Bound": confidence_intervals[0],
        "Upper Bound": confidence_intervals[1],
        "Category": category_name,
    }
    results_df = pd.DataFrame(results)
    return results_df, mean_scores, confidence_intervals
results_df, mean_scores, confidence_intervals = get_bootstrapped_score(X, y, selected_models, n_bootstrap=10)

# Test that confidence intervals make sense
assert (confidence_intervals[0] <= confidence_intervals[1]).all(), "Every lower bound must be <= upper bound."
assert ((confidence_intervals[0] <= mean_scores) & (mean_scores <= confidence_intervals[1])).all(), "Each mean score should lie within its CI."

def assign_rank(row, df=results_df):
    """
    Input:
        row : pd.Series
            A row of the DataFrame (representing a model’s metrics).
        df : pd.DataFrame (default = results_df)
            DataFrame containing model performance with 'Lower Bound' and 'Upper Bound'.

    Output:
        int : The rank of the model, defined as (# of models confidently better) + 1.
    """

    count = 0
    for _,row2 in df.iterrows():
        if row['Lower Bound'] > row2['Upper Bound']:
            count+=1

    return count


results_df['Rank'] = results_df.apply(lambda r: assign_rank(r, results_df), axis=1)
results_df = results_df.sort_values(by="Rank", ascending=True)
results_df

model_lineup = results_df.sort_values("Rank")['Model'].tolist()
avg_lengths = per_model_battles.groupby("model")["response_length"].mean().reset_index()
avg_lengths["model"] = pd.Categorical(avg_lengths["model"], categories=model_lineup, ordered=True)
avg_lengths = avg_lengths.sort_values("model").reset_index(drop=True)

# Add a numeric rank column for trendline fitting
avg_lengths["rank"] = avg_lengths.index + 1  # 1 = best, etc.

# Fit a linear trendline (polyfit) to the response length vs. rank
z = np.polyfit(avg_lengths["rank"], avg_lengths["response_length"], 1)
p = np.poly1d(z)
trendline = p(avg_lengths["rank"])

fig = go.Figure()

fig.add_trace(go.Scatter(
    x=avg_lengths["model"],
    y=avg_lengths["response_length"],
    mode='lines+markers',
    name='Avg Response Length'
))

fig.add_trace(go.Scatter(
    x=avg_lengths["model"],
    y=trendline,
    mode='lines',
    name='Trendline',
    line=dict(dash='dash', color='red')
))

fig.update_layout(
    title="Average Response Length of Models (with Trendline)",
    xaxis_title="Model (sorted by performance)",
    yaxis_title="Average Response Length",
    xaxis_tickangle=45,
    yaxis=dict(range=[0, max(avg_lengths["response_length"].max(), trendline.max()) * 1.05])  # y-axis starts at 0
)

fig.show()

def add_style_features(df):
    """
    Adds normalized style feature difference columns to the DataFrame.
    The columns added are:
      - style_bold_count
      - style_header_count
      - style_list_count
      - style_sum_assistant_tokens
    """
    def normdiff(a, b):
        denom = a + b
        return 0 if denom == 0 else (a - b) / denom

    
    style_bold = []
    style_header = []
    style_list = []
    style_tokens = []
    for idx, row in df.iterrows():

        meta=row['conv_metadata']

        bold_a=sum(meta['bold_count_a'].values())
        bold_b=sum(meta['bold_count_b'].values())

        header_a=sum(meta['header_count_a'].values())
        header_b=sum(meta['header_count_b'].values())

        list_a=sum(meta['list_count_a'].values())
        list_b=sum(meta['list_count_b'].values())

        tokens_a=meta['sum_assistant_a_tokens']
        tokens_b=meta['sum_assistant_b_tokens']

        style_bold.append(normdiff(bold_a,bold_b))
        style_header.append(normdiff(header_a,header_b))
        style_list.append(normdiff(list_a,list_b))
        style_tokens.append(normdiff(tokens_a,tokens_b))



        
        
    df["style_bold_count"] = style_bold
    df["style_header_count"] = style_header
    df["style_list_count"] = style_list
    df["style_sum_assistant_tokens"] = style_tokens
    return df

# Example usage:
selected_battles_no_ties = add_style_features(selected_battles_no_ties)

def make_pairwise_feature_df(df, models, style_feature_cols):
    """
    For each row in df, create two rows in the output:
      - One for A->B (original direction)
      - One for B->A (flipped direction)
    Each row contains:
      - question_id
      - X: model indicator vector (1 for model_a, -1 for model_b, 0 otherwise)
      - y: 1 if model_a wins, 0 if model_b wins
      - style features (from style_feature_cols)
      - direction: "A->B" or "B->A"
    Returns a new DataFrame with only these columns.
    """
    ...
    records = []
    for idx, row in df.iterrows():
        # X vector for A->B and B->A
        x_a_b=[1 if row['model_a']==model else -1 if row['model_b']==model else 0 for model in models]
        x_b_a=[-1 if row['model_a']==model else 1 if row['model_b']==model else 0 for model in models]
        # y for A->B and B->A
        y_a_b=1 if row['winner']=='model_a' else 0
        y_b_a=1 if  y_a_b==0 else 1
        # Style features for A->B, B->A
        style_ab=row[style_feature_cols].to_numpy(dtype=float)
        style_ba=-style_ab

        rec_a={
          "question_id":row['question_id'],
          "X":x_a_b,
          "y":y_a_b,
          "direction":"A->B",
        }

        for col,val in zip(style_feature_cols,style_ab):
          rec_a[col]=val

        rec_b={
          "question_id":row['question_id'],
          "X":x_b_a,
          "y":y_b_a,
          "direction":"B->A",
        }

        for col,val in zip(style_feature_cols,style_ba):
          rec_b[col]=val


        
        # Add A->B, B->A
        records.append(rec_a)
        records.append(rec_b)
        
    
    return pd.DataFrame(records)

style_feature_cols = [
    "style_bold_count",
    "style_header_count",
    "style_list_count",
    "style_sum_assistant_tokens"
]

pairwise_feature_df = make_pairwise_feature_df(selected_battles_no_ties, selected_models, style_feature_cols)
pairwise_feature_df.head(2)

def get_sc_category_results(df, selected_models, filter_mask = None,
                            category_name="Overall w/ Style Control",
                            n_bootstrap=10, style_features=style_feature_cols):
    feature_labels = selected_models + style_features
    if filter_mask is not None:
        df = df[filter_mask]

    X_id = np.vstack(df['X'].values)
    X_style = df[style_features].to_numpy()
    X_with_style = np.hstack([X_id, X_style])

    results_df, mean_scores, confidence_intervals = get_bootstrapped_score(
        X_with_style, df["y"].to_numpy(), feature_labels,
        category_name=category_name, n_bootstrap=n_bootstrap
    )

    results_df["Rank"] = results_df.apply(lambda r: assign_rank(r, results_df), axis=1)
    results_df.loc[results_df["Model"].isin(style_features), "Rank"] = -1

    # 按题目要求重新排列顺序
    cols = ['Model', 'Category', 'Average Score', 'Lower Bound', 'Upper Bound', 'Rank']
    results_df = results_df[cols]

    return results_df

results_df_style_control = get_sc_category_results(
    pairwise_feature_df, selected_models,
    category_name="Overall w/ Style Control", n_bootstrap=10
)

combined_results_df = pd.concat([results_df, results_df_style_control], ignore_index=True)
cols = ['Model', 'Category', 'Average Score', 'Lower Bound', 'Upper Bound', 'Rank']
combined_results_df = combined_results_df[cols]

fig = plot_rank_heatmap(combined_results_df, title="With and Without Style Control")
fig.show()

# TFIDF comparing winning responses to losing responses
...
# 1) Restrict to English Only and prepare assistant-only strings for A/B
# Use 'convert_asst_conversation_to_string' helper function
selected_battles_english = selected_battles_no_ties[selected_battles_no_ties['language'] == 'English'].copy()

selected_battles_english['asst_a']=selected_battles_english['conversation_a'].apply(convert_asst_conversation_to_string)
selected_battles_english['asst_b']=selected_battles_english['conversation_b'].apply(convert_asst_conversation_to_string)

# 2) Build lists of assistant-only winning/losing responses
winning_responses = selected_battles_english.apply(lambda row:row['asst_a'] if row['winner']=='model_a' else row['asst_b'],axis=1).tolist()

losing_responses = selected_battles_english.apply(lambda row:row['asst_a'] if row['winner']=='model_b' else row['asst_b'],axis=1).tolist()

# 3) Compare phrases
df_win, df_lose = tfidf_phrase_diff(winning_responses, losing_responses, "winning", "losing")
# 4) Show results
print("Top winning response phrases:")
display(df_win)
print("Top losing response phrases:")
display(df_lose)

# Design your own stylistic feature(s) that compare model A vs B on each row.
# Keep it simple (boolean presence, counts, or normalized differences) or get creative.

# Remember might want to normalize difference, if you compute counts

# TODO: Define your feature function
def YOUR_FUNC(row):
    #Input: a row with 'conversation_a' and 'conversation_b' (each is a list of {role, content} dicts).
    #Output: a single numeric feature comparing A vs B (e.g., -1/0/1, count diff, or normalized diff).
    #Hint: You probably want to inspect only assistant messages.
    # Example (placeholder): return 0
    phrases = [
        "step by step",
        "follow these steps",
        "the following steps",
        "let's break it down",
        "let us break it down",
        "for example",
        "for instance",
        "in summary",
        "to summarize"
    ]
    a_has=False
    b_has=False

    for msg in row['conversation_a']:
        if msg['role'] == 'assistant':
            text = msg['content'].lower().replace("-", " ")
            if any([p in text for p in phrases]):
                a_has=True
    for msg in row['conversation_b']:
        if msg['role'] == 'assistant':
            text = msg['content'].lower().replace("-", " ")
            if any([p in text for p in phrases]):
                b_has=True

    return int(a_has)-int(b_has)

# TODO: Apply your function to create a new column
selected_battles_no_ties.loc[:, "YOUR_FEATURE"] = selected_battles_no_ties.apply(YOUR_FUNC, axis=1)

# Choose which style features to control for in ranking.
# Start from this list and add yours below.
style_feature_cols = [
    "style_bold_count",
    "style_header_count",
    "style_list_count",
    "style_sum_assistant_tokens",
    "refusal_count",
    "YOUR_FEATURE",   # <- uncomment after you create it
]

pairwise_feature_df = make_pairwise_feature_df(selected_battles_no_ties, selected_models, style_feature_cols)

combined_results_df_new = get_sc_category_results(pairwise_feature_df,
                                 selected_models, 
                                 category_name="Overall w/ Style Control", 
                                 n_bootstrap=10, 
                                 style_features=style_feature_cols)

fig = plot_rank_heatmap(pd.concat([results_df, combined_results_df_new]), title="With and Without Style Control (New Features)", selected_models=selected_models)
fig.show()

# Create and display the plot
fig = plot_style_features(combined_results_df_new, selected_models)
fig.show()