[logseq-plugin-git:commit] 2025-06-10T08:53:29.520Z

assets/storages/logseq-citation-manager/Biblioteca-zotero.bib

@@ -38863,7 +38863,258 @@ This surveying paper’s central aspect is its comprehensive summary of the gove
 }
 
 @misc{RecSys_2025_Reproducibility_689,
-  title = {{{RecSys}}\_2025\_{{Reproducibility}}\_689}
+  title = {{{RecSys}}\_2025\_{{Reproducibility}}\_689},
+  note = {I'm reviewing a research paper and I took the following notes:
+\par
+\section{Annotazioni\\
+(10/6/2025, 10:46:31)}
+
+\par
+- “Compared to traditional methods such as collaborative filtering, LLM-based recommendation offers advantages in handling cold-start, crossdomain, and zero-shot scenarios, as well as supporting flexible input formats and generating explanations of user behavior.” (“RecSys\_2025\_Reproducibility\_689”, p. 1) \#a28ae5
+
+\par
+- “large-scale comparison of 23 prompt types across 8 public datasets and 12 LLMs.” (“RecSys\_2025\_Reproducibility\_689”, p. 1) \#e56eee
+
+\par
+- “accuracy and inference cost.” (“RecSys\_2025\_Reproducibility\_689”, p. 1) \#a28ae5
+
+\par
+- “Our results show that for cost-efficient LLMs, three types of prompts are especially effective: those that rephrase instructions, consider background knowledge, and make the reasoning process easier to follow.” (“RecSys\_2025\_Reproducibility\_689”, p. 1) \#5fb236
+
+\par
+- “high-performance LLMs, simple prompts often outperform more complex ones while reducing cost.” (“RecSys\_2025\_Reproducibility\_689”, p. 1) \#a28ae5
+
+\par
+- “Based on these findings, we provide practical suggestions for selecting prompts and LLMs depending on the required balance between accuracy and cost.” (“RecSys\_2025\_Reproducibility\_689”, p. 1) \#ffd400\\
+\mkbibemph{without knowing the task to be performed?}
+\par
+- “These methods work well when there is a large amount of training data. However, their performance decreases in cold-start situations [54, 66, 67] where the system has not seen the target items or users before, and also in cross-domain situations [22, 36, 62] where the system is applied to a different domain from its original training domain, such as recommending books after learning from movie data.” (“RecSys\_2025\_Reproducibility\_689”, p. 1) \#e56eee
+
+\par
+- “When a recommendation task is described in natural language, an LLM can return meaningful results without additional training” (“RecSys\_2025\_Reproducibility\_689”, p. 1) \#5fb236
+
+\par
+- “They can also handle item attributes that are not included in the training data and provide informative explanations for their inferences” (“RecSys\_2025\_Reproducibility\_689”, p. 1) \#5fb236
+
+\par
+- “Recommendation tasks, however, require different types of reasoning that emphasize the relationship between users and items.” (“RecSys\_2025\_Reproducibility\_689”, p. 1) \#5fb236
+
+\par
+- “Research on recommendation has mostly focused on prompt types proposed in the RecSys field” (“RecSys\_2025\_Reproducibility\_689”, p. 1) \#5fb236
+
+\par
+- “Our study compares 23 prompt types, 8 real-world datasets, and 12 LLMs,” (“RecSys\_2025\_Reproducibility\_689”, p. 1) \#5fb236
+
+\par
+- “We used hypothesis testing and linear mixed-effects models to identify which prompts led to better recommendation accuracy.” (“RecSys\_2025\_Reproducibility\_689”, p. 2) \#5fb236
+
+\par
+- “In contrast, step-by-step reasoning [23], which is often said to be effective in NLP, did not lead to higher accuracy in recommendation.” (“RecSys\_2025\_Reproducibility\_689”, p. 2) \#5fb236
+
+\par
+- “complex prompt designs.” (“RecSys\_2025\_Reproducibility\_689”, p. 2) \#ffd400\\
+\mkbibemph{what does it mean? What are the dimensions to characterize complexity?}
+\par
+- “These results suggest that adding more reasoning does not always improve performance in recommendation tasks and may sometimes reduce accuracy” (“RecSys\_2025\_Reproducibility\_689”, p. 2) \#5fb236
+
+\par
+- “NLP-Based Recommendation.” (“RecSys\_2025\_Reproducibility\_689”, p. 2) \#2ea8e5
+
+\par
+- “prompt design influences recommendation accuracy, we conducted large-scale experiments using eight datasets, two user types, 12 LLMs, and standard ranking metrics.” (“RecSys\_2025\_Reproducibility\_689”, p. 2) \#a28ae5
+
+\par
+- “In each dataset, users with 6 to 11 interactions were labeled as light, and those with 31 or more as heavy.” (“RecSys\_2025\_Reproducibility\_689”, p. 2) \#5fb236
+
+\par
+- “We evaluated 12 LLMs from OpenAI, Meta, AWS, Microsoft, and Anthropic. Based on cost and internal behavior, we classified the LLMs into three: low-cost, high-cost, and reasoning models” (“RecSys\_2025\_Reproducibility\_689”, p. 2) \#5fb236
+
+\par
+- “All OpenAI models were accessed through the official API. Models from Meta, AWS, and Anthropic were accessed via Amazon Bedrock.” (“RecSys\_2025\_Reproducibility\_689”, p. 3) \#5fb236
+
+\par
+- “Standardized Phrases.” (“RecSys\_2025\_Reproducibility\_689”, p. 3) \#2ea8e5
+
+\par
+- “In the numerical experiments, each prompt produced a ranking of 10 candidate items.” (“RecSys\_2025\_Reproducibility\_689”, p. 4) \#5fb236
+
+\par
+- “We preserved the order of these items and randomly ranked the rest to complete a full list of 10.” (“RecSys\_2025\_Reproducibility\_689”, p. 4) \#ffd400\\
+\mkbibemph{The randomly ranked missing items to complete the full list of 10 represents a potential bias, isn't it?}
+\par
+- “After five failures, we used a fully random ranking.” (“RecSys\_2025\_Reproducibility\_689”, p. 4) \#ffd400\\
+\mkbibemph{This also can be a potential bias.}
+\par
+- “effectiveness of the 21 prompt types introduced in Section 3.3 in terms of recommendation accuracy” (“RecSys\_2025\_Reproducibility\_689”, p. 4) \#5fb236
+
+\par
+- “goal is to understand which prompts are effective (RQ1), under what conditions (RQ2), and why (RQ3).” (“RecSys\_2025\_Reproducibility\_689”, p. 5) \#5fb236
+
+\par
+- “baseline” (“RecSys\_2025\_Reproducibility\_689”, p. 5) \#ffd400\\
+\mkbibemph{which one?}
+\par
+- “This analysis helps identify which prompts are effective for specific LLMs.” (“RecSys\_2025\_Reproducibility\_689”, p. 5) \#a28ae5
+
+\par
+- “valuation Method.” (“RecSys\_2025\_Reproducibility\_689”, p. 5) \#2ea8e5
+
+\par
+- “relative improvement of each prompt using r p = (mp − mBaseline)/mBaseline, where mp is the average score for prompt p” (“RecSys\_2025\_Reproducibility\_689”, p. 5) \#5fb236
+
+\par
+- “Stable prompts across datasets and LLMs.” (“RecSys\_2025\_Reproducibility\_689”, p. 5) \#2ea8e5
+
+\par
+- “Smaller effects in low-performing LLMs” (“RecSys\_2025\_Reproducibility\_689”, p. 5) \#2ea8e5
+
+\par
+- “Performance variation depending on LLMs” (“RecSys\_2025\_Reproducibility\_689”, p. 5) \#2ea8e5
+
+\par
+- “Limited effects of NLP-style prompts” (“RecSys\_2025\_Reproducibility\_689”, p. 5) \#2ea8e5
+
+\par
+- “linear mixed-effects model (LMEM) to examine how each prompt is effective while controlling for factors such as the LLM, user, and evaluation metri” (“RecSys\_2025\_Reproducibility\_689”, p. 5) \#5fb236
+
+\par
+- “While the previous analysis in Section 4.1 showed that performance varied depending on the LLM, this analysis aims to find prompts that are effective regardless of the LLM or dataset” (“RecSys\_2025\_Reproducibility\_689”, p. 5) \#a28ae5
+
+\par
+- “Since these two LLMs showed relatively strong performance among the low-cost models, their results may have exerted a stronger influence on the LMEM estimate” (“RecSys\_2025\_Reproducibility\_689”, p. 5) \#5fb236
+
+\par
+- “Prompts such as Re-Reading, RolePlay-User, SummarizeItem, and Echo showed lower accuracy in the LMEM analysis. As shown in Table 7,” (“RecSys\_2025\_Reproducibility\_689”, p. 5) \#e56eee
+
+\par
+- “, if an LLM failed to return any ranking and all five retries also failed, we completed it by random ranking” (“RecSys\_2025\_Reproducibility\_689”, p. 5) \#ffd400\\
+\mkbibemph{Exactly. See my previous comment about this potential bias.}
+\par
+- “here the LLM ignored the instruction in task\_inst to return a full ranking” (“RecSys\_2025\_Reproducibility\_689”, p. 6) \#5fb236
+
+\par
+- “We summarize the findings from this section. In Section 4.1, we compared recommendation accuracy across different prompts for each LLM and dataset. Prompts such as Rephrase, Step-Back, Explain, Recency-Focused, and Summarize-User improved accuracy under many conditions” (“RecSys\_2025\_Reproducibility\_689”, p. 6) \#5fb236
+
+\par
+- “Based on these findings, we conclude that in cost-efficient LLMs such as gpt-4.1-mini and llama3.3-70b, prompts that clarify the task context and goal, such as ReAct, Step-Back, and Rephrase, are especially effective for recommendation.” (“RecSys\_2025\_Reproducibility\_689”, p. 7) \#5fb236
+
+\par
+- “we identified several prompts that improved recommendation accuracy. In this section, we explore whether additional improvement is possible by combining prompts, applying metalevel techniques, or using more advanced LLMs.” (“RecSys\_2025\_Reproducibility\_689”, p. 7) \#a28ae5
+
+\par
+- “We apply Self-Refine and Self-Consistency to Rephrase, ReAct, and Step-Back, resulting in six meta-prompt variants.” (“RecSys\_2025\_Reproducibility\_689”, p. 7) \#5fb236
+
+\par
+- “Table 12” (“RecSys\_2025\_Reproducibility\_689”, p. 7) \#ffd400\\
+\mkbibemph{We are missing the data related to the application of Rephrase only, isn't it?}
+\par
+- “Rephrase, ReAct, and Step-Back are already effective without additional techniques. Since additional instructions increased inference cost but did not meaningfully improve accuracy, this conclusion seems reasonable.” (“RecSys\_2025\_Reproducibility\_689”, p. 7) \#ffd400\\
+\mkbibemph{WHat can we say about sentences like this? What can we say about the external validity of the performed experiments? How could we reuse such insights?}
+\par
+- “As another way to improve recommendation accuracy, we consider using more powerful LLMs. In this section, we examine whether the prompts Rephrase, ReAct, and Step-Back, which showed strong performance, are still effective with stronger models.” (“RecSys\_2025\_Reproducibility\_689”, p. 7) \#5fb236
+
+\par
+- “Accuracy decreasing from overthinking” (“RecSys\_2025\_Reproducibility\_689”, p. 8) \#a28ae5
+
+\par
+- “These observations suggest that when reasoning steps become complicated, the LLM may become confused, which is referred to as the overthinking problem” (“RecSys\_2025\_Reproducibility\_689”, p. 8) \#a28ae5
+
+\par
+- “To avoid these, we recommend checking the behavior of prompts in advance before deploying them in recommendation tasks” (“RecSys\_2025\_Reproducibility\_689”, p. 8) \#5fb236
+
+\par
+- “Best Option for Accuracy” (“RecSys\_2025\_Reproducibility\_689”, p. 8) \#2ea8e5
+
+\par
+- “Considering the risk that unnecessary modifications may lead to reduce accuracy, we conclude that this simple configuration is the most reasonable choice” (“RecSys\_2025\_Reproducibility\_689”, p. 8) \#5fb236
+
+\par
+- “Best option for cost efficiency. For second-tier accuracy around 0.55, using gpt-4.1-mini or llama3.3-70b with one of Rephrase, ReAct, or Step-Back was cost-efficient.” (“RecSys\_2025\_Reproducibility\_689”, p. 8) \#e56eee
+
+\par
+- “This study examined the effect of prompt design on personalized recommendation using LLMs. We compared 23 prompts across 8 datasets and 12 LLMs in a systematic evaluation. We also tested combinations of prompts and meta-level techniques.” (“RecSys\_2025\_Reproducibility\_689”, p. 8) \#e56eee
+
+\par
+- “We found that prompts that restructure user context or guide the model through structured reasoning, such as Rephrase, StepBack, and ReAct, were effective with cost-efficient LLMs like gpt-4.1-mini and llama3.3-70b” (“RecSys\_2025\_Reproducibility\_689”, p. 8) \#e56eee
+
+\par
+- “general-purpose prompting techniques often used in NLP, such as RolePlay, Emotion, and Step-by-Step, did not always improve recommendation accuracy and reduced performance in some cases.” (“RecSys\_2025\_Reproducibility\_689”, p. 8) \#e56eee
+
+\par
+- “For highperformance models like claude-3.7-sonnet, the simple Baseline prompt achieved high accuracy without increasing inference cost” (“RecSys\_2025\_Reproducibility\_689”, p. 8) \#e56eee
+
+\par
+COnsider that those that are tagget with \#5fb236 are just highlights, those that are tagged with \#e56eee and \#a28ae5 are imporant sentences. Please pay attention instead to the notes that are tagged with \#ffd400. Those that are tagged with \#ff6666 are typos or errors. Could you please draft a review by organizing it as follows:
+\par
+\textbf{Overall Presentation.}*~Please reflect on the overall quality of the presentation of this contribution. Poor – The paper is difficult to follow due to unclear writing, lack of structure, or numerous grammatical and typographical errors. Figures, tables, and explanations are inadequate. Below Average – The main ideas are present but not clearly communicated. Writing may be inconsistent, and the structure could be improved. Some figures or tables may be unclear. Adequate – The paper is understandable but could benefit from better clarity, organization, or formatting. Some sections may be overly complex or underdeveloped. Well-Presented – The paper is clearly written, well-structured, and easy to follow. Figures, tables, and explanations support the content. Minor improvements could enhance readability. Excellent – The paper is exceptionally well-written, logically structured, and highly engaging. All figures, tables, and explanations are clear, precise, and enhance comprehension significantly.
+\par
+5: Excellent
+\par
+4: Well-Presented
+\par
+3: Adequate
+\par
+2: Below Average
+\par
+1: Poor
+\par
+\textbf{Technical Soundness.}*~How technically sound is the research presented in this contribution? Fundamentally Flawed – The research contains critical errors, invalid assumptions, or methodological weaknesses that undermine its conclusions. Weak Technical Rigor – Some aspects of the methodology are questionable. Adequate Technical Soundness – The approach is generally valid, but there may be minor methodological weaknesses, limited evaluation, or room for improvement in execution. Strong Technical Soundness – The research is well-executed, with a solid methodology, and appropriate evaluation. Any limitations are acknowledged and justified. Excellent – This contribution is methodologically impeccable.
+\par
+5: Excellent
+\par
+4: Strong Technical Soundness
+\par
+3: Adequate Technical Soundness
+\par
+2: Weak Technical Rigor
+\par
+1: Fundamentally Flawed
+\par
+\textbf{Reproducibility of Methods.}*~Please reflect on the reproducibility of the work presented in this contribution. Not Reproducible – Key details of the methodology are missing or unclear, making it impossible for others to replicate the work. No code/data/protocols are provided. Weak Reproducibility – Some methodological details are provided, but important steps, parameters, or implementation details are missing. Limited access to code/data/protocols. Moderate Reproducibility – The methodology is described sufficiently for partial replication, but some ambiguity remains. code/data/protocols may be available but not fully documented. Highly Reproducible – The methodology is well-documented, and replication is feasible with the provided details. code/data/protocols are available and reasonably well-organized. Fully Reproducible – The study is meticulously documented with complete details, ensuring full reproducibility. code/data/protocols are openly available and well-structured for easy replication.
+\par
+5: Fully Reproducible
+\par
+4: Highly Reproducible
+\par
+3: Moderate
+\par
+2: Weak
+\par
+1: Not Reproducible
+\par
+\textbf{Impact.}*~Please reflect on whether the ideas and/or results presented will have an impact on the community. Minimal Impact – The ideas or results have little to no influence on the field. They may be too incremental, lack practical relevance, or fail to offer meaningful insights. Limited Impact – The work provides some useful findings but is unlikely to influence future research or applications. Moderate Impact – The contribution is valuable and may inspire further research or applications in certain areas, though its broader significance remains limited. High Impact – The ideas or results have strong potential to shape future research, influence practices, or contribute meaningfully to advancements in the field. Transformative Impact – The work introduces groundbreaking ideas or results that could redefine current understanding, open new research directions, or have significant real-world applications.
+\par
+5: Transformative
+\par
+4: High
+\par
+3: Moderate
+\par
+2: Limited
+\par
+1: Minimal
+\par
+\textbf{Artifacts- . Are the paper artifacts (source code, data) available and well documented?”.}*~Consider paper artifacts (source code, data) and state whether they are available and well documented
+\par
+\textbf{Artifacts. Are the artifacts executable by following the instructions provided in the documentation?.}~Consider whether the artifacts are executable by following the instructions provided in the documentation. Note that we do not expect the reviewers to run the full set of experiments, especially if they are expensive or rely on specialized hardware, rather we ask you to assess if the experimental pipeline can be started and is executable. Minor issues are acceptable if they can be fixed easily.
+\par
+\textbf{Review.}*~Please provide a review of the paper, offering general comments to authors, along with justification from the different aspects you have rated this contribution (relevance, originality, technical soundness, impact, overall presentation, coverage of related work, and reproducibility). In other words, when reviewing, consider how well the paper builds on prior work, the soundness of its methodology, the strength of its evidence and claims, and the clarity of its presentation. Also, assess its practical relevance, ethical considerations, and whether it provides enough detail for replication.
+\par
+\textbf{Showstopper.}*~To streamline the review process, please list any key issues where a response from the authors during the rebuttal phase would be essential for your evaluation of this contribution.
+\par
+\textbf{Overall Evaluation.}*~You have given detailed feedback to the authors. Now, we ask for your final recommendation along with a brief justification. Both the recommendation score and the review text are required. Remember that when making your final recommendation, you should focus primarily on whether this work makes a strong and original contribution to the field of Recommender Systems. You should also consider the impact of the presented outcomes and the rigor of the research methodology.
+\par
+3: Strong accept
+\par
+2: Accept
+\par
+1: Weak accept
+\par
+-1: Weak reject
+\par
+-2: Reject
+\par
+-3: Strong reject}
 }
 
 @misc{RecSys_2025_Reproducibility_711,