<?xml version="1.0" encoding="UTF-8"?>
<!DOCTYPE article PUBLIC "-//NLM//DTD Journal Publishing DTD v2.0 20040830//EN" "http://dtd.nlm.nih.gov/publishing/2.0/journalpublishing.dtd">
<article xmlns:xlink="http://www.w3.org/1999/xlink" article-type="review-article" dtd-version="2.0">
  <front>
    <journal-meta>
      <journal-id journal-id-type="publisher-id">JMH</journal-id>
      <journal-id journal-id-type="nlm-ta">JMIR Ment Health</journal-id>
      <journal-title>JMIR Mental Health</journal-title>
      <issn pub-type="epub">2368-7959</issn>
      <publisher>
        <publisher-name>JMIR Publications</publisher-name>
        <publisher-loc>Toronto, Canada</publisher-loc>
      </publisher>
    </journal-meta>
    <article-meta>
      <article-id pub-id-type="publisher-id">v11i1e58974</article-id>
      <article-id pub-id-type="pmid">39250799</article-id>
      <article-id pub-id-type="doi">10.2196/58974</article-id>
      <article-categories>
        <subj-group subj-group-type="heading">
          <subject>Review</subject>
        </subj-group>
        <subj-group subj-group-type="article-type">
          <subject>Review</subject>
        </subj-group>
      </article-categories>
      <title-group>
        <article-title>Empathic Conversational Agent Platform Designs and Their Evaluation in the Context of Mental Health: Systematic Review</article-title>
      </title-group>
      <contrib-group>
        <contrib contrib-type="editor">
          <name>
            <surname>Torous</surname>
            <given-names>John</given-names>
          </name>
        </contrib>
      </contrib-group>
      <contrib-group>
        <contrib contrib-type="reviewer">
          <name>
            <surname>Okoro</surname>
            <given-names>Yvonne</given-names>
          </name>
        </contrib>
        <contrib contrib-type="reviewer">
          <name>
            <surname>Upadhyaya</surname>
            <given-names>Pulakesh</given-names>
          </name>
        </contrib>
        <contrib contrib-type="reviewer">
          <name>
            <surname>Teferra</surname>
            <given-names>Bazen Gashaw</given-names>
          </name>
        </contrib>
        <contrib contrib-type="reviewer">
          <name>
            <surname>Bagewadi Ellur</surname>
            <given-names>Mallikarjuna</given-names>
          </name>
        </contrib>
      </contrib-group>
      <contrib-group>
        <contrib id="contrib1" contrib-type="author" corresp="yes">
          <name name-style="western">
            <surname>Sanjeewa</surname>
            <given-names>Ruvini</given-names>
          </name>
          <degrees>BSc</degrees>
          <xref rid="aff1" ref-type="aff">1</xref>
          <address>
            <institution>School of Health Sciences</institution>
            <institution>Swinburne University of Technology</institution>
            <addr-line>PO Box 218</addr-line>
            <addr-line>John Street</addr-line>
            <addr-line>Hawthorn, 3122</addr-line>
            <country>Australia</country>
            <phone>61 422587030</phone>
            <email>rsanjeewa@swin.edu.au</email>
          </address>
          <ext-link ext-link-type="orcid">https://orcid.org/0009-0008-8301-6994</ext-link>
        </contrib>
        <contrib id="contrib2" contrib-type="author">
          <name name-style="western">
            <surname>Iyer</surname>
            <given-names>Ravi</given-names>
          </name>
          <degrees>PhD</degrees>
          <xref rid="aff1" ref-type="aff">1</xref>
          <ext-link ext-link-type="orcid">https://orcid.org/0000-0001-7699-0846</ext-link>
        </contrib>
        <contrib id="contrib3" contrib-type="author">
          <name name-style="western">
            <surname>Apputhurai</surname>
            <given-names>Pragalathan</given-names>
          </name>
          <degrees>PhD</degrees>
          <xref rid="aff1" ref-type="aff">1</xref>
          <ext-link ext-link-type="orcid">https://orcid.org/0000-0003-1667-6315</ext-link>
        </contrib>
        <contrib id="contrib4" contrib-type="author">
          <name name-style="western">
            <surname>Wickramasinghe</surname>
            <given-names>Nilmini</given-names>
          </name>
          <degrees>PhD</degrees>
          <xref rid="aff2" ref-type="aff">2</xref>
          <ext-link ext-link-type="orcid">https://orcid.org/0000-0002-1314-8843</ext-link>
        </contrib>
        <contrib id="contrib5" contrib-type="author">
          <name name-style="western">
            <surname>Meyer</surname>
            <given-names>Denny</given-names>
          </name>
          <degrees>PhD</degrees>
          <xref rid="aff1" ref-type="aff">1</xref>
          <ext-link ext-link-type="orcid">https://orcid.org/0000-0002-9902-0858</ext-link>
        </contrib>
      </contrib-group>
      <aff id="aff1">
        <label>1</label>
        <institution>School of Health Sciences</institution>
        <institution>Swinburne University of Technology</institution>
        <addr-line>Hawthorn</addr-line>
        <country>Australia</country>
      </aff>
      <aff id="aff2">
        <label>2</label>
        <institution>School of Computing, Engineering and Mathematical Sciences</institution>
        <institution>La Trobe University</institution>
        <addr-line>Bundoora</addr-line>
        <country>Australia</country>
      </aff>
      <author-notes>
        <corresp>Corresponding Author: Ruvini Sanjeewa <email>rsanjeewa@swin.edu.au</email></corresp>
      </author-notes>
      <pub-date pub-type="collection">
        <year>2024</year>
      </pub-date>
      <pub-date pub-type="epub">
        <day>9</day>
        <month>9</month>
        <year>2024</year>
      </pub-date>
      <volume>11</volume>
      <elocation-id>e58974</elocation-id>
      <history>
        <date date-type="received">
          <day>30</day>
          <month>3</month>
          <year>2024</year>
        </date>
        <date date-type="rev-request">
          <day>6</day>
          <month>5</month>
          <year>2024</year>
        </date>
        <date date-type="rev-recd">
          <day>1</day>
          <month>7</month>
          <year>2024</year>
        </date>
        <date date-type="accepted">
          <day>2</day>
          <month>7</month>
          <year>2024</year>
        </date>
      </history>
      <copyright-statement>©Ruvini Sanjeewa, Ravi Iyer, Pragalathan Apputhurai, Nilmini Wickramasinghe, Denny Meyer. Originally published in JMIR Mental Health (https://mental.jmir.org), 09.09.2024.</copyright-statement>
      <copyright-year>2024</copyright-year>
      <license license-type="open-access" xlink:href="https://creativecommons.org/licenses/by/4.0/">
        <p>This is an open-access article distributed under the terms of the Creative Commons Attribution License (https://creativecommons.org/licenses/by/4.0/), which permits unrestricted use, distribution, and reproduction in any medium, provided the original work, first published in JMIR Mental Health, is properly cited. The complete bibliographic information, a link to the original publication on https://mental.jmir.org/, as well as this copyright and license information must be included.</p>
      </license>
      <self-uri xlink:href="https://mental.jmir.org/2024/1/e58974" xlink:type="simple"/>
      <abstract>
        <sec sec-type="background">
          <title>Background</title>
          <p>The demand for mental health (MH) services in the community continues to exceed supply. At the same time, technological developments make the use of artificial intelligence–empowered conversational agents (CAs) a real possibility to help fill this gap.</p>
        </sec>
        <sec sec-type="objective">
          <title>Objective</title>
          <p>The objective of this review was to identify existing empathic CA design architectures within the MH care sector and to assess their technical performance in detecting and responding to user emotions in terms of classification accuracy. In addition, the approaches used to evaluate empathic CAs within the MH care sector in terms of their acceptability to users were considered. Finally, this review aimed to identify limitations and future directions for empathic CAs in MH care.</p>
        </sec>
        <sec sec-type="methods">
          <title>Methods</title>
          <p>A systematic literature search was conducted across 6 academic databases to identify journal articles and conference proceedings using search terms covering 3 topics: “conversational agents,” “mental health,” and “empathy.” Only studies discussing CA interventions for the MH care domain were eligible for this review, with both textual and vocal characteristics considered as possible data inputs. Quality was assessed using appropriate risk of bias and quality tools.</p>
        </sec>
        <sec sec-type="results">
          <title>Results</title>
          <p>A total of 19 articles met all inclusion criteria. Most (12/19, 63%) of these empathic CA designs in MH care were machine learning (ML) based, with 26% (5/19) hybrid engines and 11% (2/19) rule-based systems. Among the ML-based CAs, 47% (9/19) used neural networks, with transformer-based architectures being well represented (7/19, 37%). The remaining 16% (3/19) of the ML models were unspecified. Technical assessments of these CAs focused on response accuracies and their ability to recognize, predict, and classify user emotions. While single-engine CAs demonstrated good accuracy, the hybrid engines achieved higher accuracy and provided more nuanced responses. Of the 19 studies, human evaluations were conducted in 16 (84%), with only 5 (26%) focusing directly on the CA’s empathic features. All these papers used self-reports for measuring empathy, including single or multiple (scale) ratings or qualitative feedback from in-depth interviews. Only 1 (5%) paper included evaluations by both CA users and experts, adding more value to the process.</p>
        </sec>
        <sec sec-type="conclusions">
          <title>Conclusions</title>
          <p>The integration of CA design and its evaluation is crucial to produce empathic CAs. Future studies should focus on using a clear definition of empathy and standardized scales for empathy measurement, ideally including expert assessment. In addition, the diversity in measures used for technical assessment and evaluation poses a challenge for comparing CA performances, which future research should also address. However, CAs with good technical and empathic performance are already available to users of MH care services, showing promise for new applications, such as helpline services.</p>
        </sec>
      </abstract>
      <kwd-group>
        <kwd>conversational agents</kwd>
        <kwd>chatbots</kwd>
        <kwd>virtual assistants</kwd>
        <kwd>empathy</kwd>
        <kwd>emotionally aware</kwd>
        <kwd>mental health</kwd>
        <kwd>mental well-being</kwd>
      </kwd-group>
    </article-meta>
  </front>
  <body>
    <sec sec-type="introduction">
      <title>Introduction</title>
      <sec>
        <title>Background</title>
        <p>An escalation in mental health (MH) diagnoses in the community, inadequate facilities, and a MH care workforce that does not meet demand are placing extraordinary pressures on an already strained system [<xref ref-type="bibr" rid="ref1">1</xref>]. This service gap creates a significant opportunity for MH care interventions, enhanced using recent advances in modern technologies. Conversational agent (CA) platforms using artificial intelligence (AI) via machine learning (ML) techniques have emerged within the MH care domain, providing additional functionalities and support to address this gap [<xref ref-type="bibr" rid="ref2">2</xref>]. Examples of CAs that use ML include Woebot, providing cognitive behavioral therapy [<xref ref-type="bibr" rid="ref3">3</xref>]; Wysa, providing MH support by checking depressive symptoms [<xref ref-type="bibr" rid="ref4">4</xref>]; Saarthi, trained to provide personalized and empathic support to patients via therapeutic techniques [<xref ref-type="bibr" rid="ref5">5</xref>]; and Empathetic Research IoT Network, a chatbot that provides access to MH resources for students in need [<xref ref-type="bibr" rid="ref6">6</xref>]. However, the lack of acceptance of CAs in the MH domain remains a barrier to the uptake of these innovations, and the lack of empathy often displayed by CAs contributes to end-user mistrust [<xref ref-type="bibr" rid="ref7">7</xref>].</p>
        <p>Empathy in patient care has been defined by the World Health Organization as an understanding of the patient’s experiences, concerns, and perspectives, combined with a capacity to communicate this understanding and an intention to help [<xref ref-type="bibr" rid="ref8">8</xref>]. Counselor empathy is an essential feature that enhances therapeutic outcomes for patients and can be measured via therapeutic alliance [<xref ref-type="bibr" rid="ref9">9</xref>,<xref ref-type="bibr" rid="ref10">10</xref>]. The same is true for CA-human interactions, where empathy exhibited by a CA system helps build rapport, encouraging users to more frequently engage with the CA system [<xref ref-type="bibr" rid="ref11">11</xref>]. Contextual awareness, which allows CAs to respond to a user’s current emotional situation when suggesting appropriate interventions, also facilitates empathic CA communication [<xref ref-type="bibr" rid="ref12">12</xref>]. Both trustworthiness of the CA (as perceived by the user) and contextual awareness of the user’s situation (as detected by the CA) are, therefore, important considerations when building an empathic CA. Empathy serves to enhance the bidirectional interaction between the CA and the end user [<xref ref-type="bibr" rid="ref13">13</xref>].</p>
        <p>Assessment of the effectiveness of CA platforms has received little attention in the MH care sector [<xref ref-type="bibr" rid="ref14">14</xref>]. For the impact of these systems to be fully realized, these platforms need to meet the requirements of end users, which suggests a key role for lived experience and coproduction. The validity and reliability of these new digital technologies also need to be reviewed by MH care decision-makers and professionals to ensure successful integration in the sector [<xref ref-type="bibr" rid="ref15">15</xref>]. In addition, evaluations need to assess the ability of such platforms to reduce symptoms of mental illness [<xref ref-type="bibr" rid="ref16">16</xref>] while also enhancing user well-being and ensuring that patients feel understood [<xref ref-type="bibr" rid="ref13">13</xref>]. However, any such evaluation needs to be conducted in the context of the role envisaged for the CA, considering the success of the bidirectional interaction described earlier.</p>
        <p>While there are existing reviews exploring the efficacy of CAs designed for MH care [<xref ref-type="bibr" rid="ref10">10</xref>,<xref ref-type="bibr" rid="ref17">17</xref>,<xref ref-type="bibr" rid="ref18">18</xref>], to our knowledge, this is the first review to specifically examine how these empathic CAs are designed and evaluated. A comprehensive systematic review and meta-analysis of AI-based CAs for promoting MH was conducted by Li et al [<xref ref-type="bibr" rid="ref17">17</xref>], with a focus on the intervention and technical characteristics of effective CAs. The effectiveness of the CA designs was captured through user feedback. The meta-analysis explored the role of the CA, AI techniques, and delivery platforms that contributed to the success of these designs. In a similar review, Gaffney et al [<xref ref-type="bibr" rid="ref18">18</xref>] targeted CA interventions for treating MH problems, with a specific focus on user experience outcomes as measures of efficacy. Another such study explored the evidence of effectiveness with regard to improving symptoms of MH conditions [<xref ref-type="bibr" rid="ref19">19</xref>]. A critical finding of this review was that empathic response and personalization were significant facilitators of efficacy in these systems. However, the incorporation of this crucial empathy component within CAs has not been studied in any depth within the MH sector. Existing reviews have tended to focus on the inability of CAs to respond to unexpected user inputs rather than their ability to demonstrate empathy [<xref ref-type="bibr" rid="ref19">19</xref>].</p>
      </sec>
      <sec>
        <title>Objectives</title>
        <p>This review aimed to assess the types of CA designs found in the MH care sector that are specifically tailored to convey empathy. It also aimed to describe the methods used to evaluate these empathic designs from a technical and implementation perspective. Therefore, this review considered how empathy has been engineered and the limitations identified with its use by a CA from a human perspective. There were three objectives: (1) to identify existing empathic CA design architectures within the MH care sector and to assess their technical performance in detecting and responding to user emotions appropriately; (2) to describe the approaches used to evaluate empathic CAs within the MH care sector in terms of their acceptability to users; and (3) to identify limitations and future directions for empathic CAs in MH care.</p>
      </sec>
    </sec>
    <sec sec-type="methods">
      <title>Methods</title>
      <sec>
        <title>Database Search</title>
        <p>A systematic literature search was conducted across 6 academic databases (Web of Science; Scopus; EBSCOhost: Academic Search Complete; CINAHL Complete; Computers and Applied Sciences Complete; and IEEE Xplore) for journal articles and conference proceedings from January 1, 2010, to September 30, 2023. The period of data capture dates from the time when AI-informed CA technology emerged as a distinct area of research [<xref ref-type="bibr" rid="ref20">20</xref>], and conference proceedings were included to ensure that the most recent studies could be included.</p>
        <p>The search terms covered 3 topics: “conversational agents,” “mental health,” and “empathy.” Possible keywords were broadened using synonyms for each topic, pilot searching of existing literature, and discussion among research team members. Boolean operators combined different keywords and their synonyms to establish the final search strategy. Wildcards were included (eg, empath* = empathic). Medical Subject Heading terms were used where appropriate. An example of the search syntax is available in <xref ref-type="supplementary-material" rid="app1">Multimedia Appendix 1</xref> [<xref ref-type="bibr" rid="ref4">4</xref>-<xref ref-type="bibr" rid="ref6">6</xref>,<xref ref-type="bibr" rid="ref21">21</xref>-<xref ref-type="bibr" rid="ref36">36</xref>].</p>
      </sec>
      <sec>
        <title>Eligibility Criteria</title>
        <p>Publications discussing CA interventions for the MH care domain were eligible for the review. There were no restrictions on research design (eg, observational designs and narrative review). This review considered both textual and vocal modes of interaction with the CA. Publications were included if they referred to CA empathy or related terms (eg, emotional intelligence, emotional awareness, and compassion). Publications that did not feature a methodology section that detailed CA design, types of data sets, and participants were excluded. Systematic reviews, scoping reviews, and meta-analyses were excluded. Publications that used data inputs other than text and vocal cues (eg, facial recognition) were also excluded. <xref ref-type="supplementary-material" rid="app1">Multimedia Appendix 1</xref> provides the full-text screening checklist.</p>
      </sec>
      <sec>
        <title>Screening</title>
        <p>Eligible references were exported to the EndNote (version 20; Clarivate) software [<xref ref-type="bibr" rid="ref37">37</xref>], where duplicates were removed. The first author (RS) conducted the title and abstract search, mapping against the eligibility criteria. A full-text screening was then performed by the first author and by 2 other authors, DM and RI, independently. Any disagreements on full-text screening were discussed, and an agreement was reached before proceeding. <xref rid="figure1" ref-type="fig">Figure 1</xref> illustrates the PRISMA (Preferred Reporting Items for Systematic Reviews and Meta-Analyses) flowchart describing the screening process. PRISMA checklist is reported in <xref ref-type="supplementary-material" rid="app2">Multimedia Appendix 2</xref>.</p>
        <p>Data including details on the study designs, how empathy was evaluated, and the types of CA architectures used were extracted to obtain a summary of all findings (<xref ref-type="supplementary-material" rid="app1">Multimedia Appendix 1</xref>).</p>
        <fig id="figure1" position="float">
          <label>Figure 1</label>
          <caption>
            <p>PRISMA (Preferred Reporting Items for Systematic Reviews and Meta-Analyses) procedure applied. CA: conversational agent.</p>
          </caption>
          <graphic xlink:href="mental_v11i1e58974_fig1.png" alt-version="no" mimetype="image" position="float" xlink:type="simple"/>
        </fig>
      </sec>
      <sec>
        <title>Quality Assessment</title>
        <p>The Joanna Briggs Institute critical appraisal tool was used to assess the methodological quality of the papers shortlisted while also considering the extent to which each study addressed the possibility of bias in design, conduct, and analysis [<xref ref-type="bibr" rid="ref38">38</xref>]. This appraisal tool was specifically designed for the assessment of the variety of study designs encountered in this systematic review. Decisional criteria were answered with <italic>yes, no, unclear, or not applicable</italic>. The proportion of <italic>yes</italic> responses relative to the total number of assessment questions was used for quality assessment purposes. Separate quality assessments were conducted for publications that included a description of the implementation as well as the design of the CA platform and for publications that included only a description of the design.</p>
      </sec>
      <sec>
        <title>Risk of Bias</title>
        <p>Risk of bias was assessed using the revised Cochrane risk-of-bias tool for randomized trials This included risks of bias due to randomization, deviations from the intended intervention, missing data, the measurement of outcomes, and the selection of results. The risk of bias in nonrandomized studies of interventions tool was used to evaluate the nonrandomized studies.</p>
      </sec>
    </sec>
    <sec sec-type="results">
      <title>Results</title>
      <sec>
        <title>Overview</title>
        <p>A total of 19 studies met all the inclusion criteria. The study characteristics are summarized in <xref ref-type="table" rid="table1">Table 1</xref>.</p>
        <table-wrap position="float" id="table1">
          <label>Table 1</label>
          <caption>
            <p>Study characteristics.</p>
          </caption>
          <table width="1000" cellpadding="5" cellspacing="0" border="1" rules="groups" frame="hsides">
            <col width="130"/>
            <col width="130"/>
            <col width="150"/>
            <col width="160"/>
            <col width="180"/>
            <col width="100"/>
            <col width="150"/>
            <thead>
              <tr valign="top">
                <td>Study</td>
                <td>CA<sup>a</sup></td>
                <td>Training database</td>
                <td>Aim of the study</td>
                <td>Evaluation measures for detecting and responding to user emotions</td>
                <td>Mode of exchange</td>
                <td>Analysis model for generating empathic responses</td>
              </tr>
            </thead>
            <tbody>
              <tr valign="top">
                <td>Jiang et al [<xref ref-type="bibr" rid="ref21">21</xref>], 2022</td>
                <td>Replika</td>
                <td>14 Chinese female users (aged 19-26 years)</td>
                <td>Explore types of mediated empathy that occur in human-AI<sup>b</sup> interactions</td>
                <td>In-depth interviews and survey results: user ratings of empathy</td>
                <td>Text and voice</td>
                <td>Transformer architecture</td>
              </tr>
              <tr valign="top">
                <td>Brocki et al [<xref ref-type="bibr" rid="ref22">22</xref>], 2023</td>
                <td>Serena</td>
                <td>Trained on “Pushshift” Reddit data set and tested on psychotherapy transcript</td>
                <td>Help improve outcomes of counseling by lowering barriers to access</td>
                <td>Survey results: user ratings of engagement and helpfulness</td>
                <td>Text</td>
                <td>Transformer architecture</td>
              </tr>
              <tr valign="top">
                <td>Persons et al [<xref ref-type="bibr" rid="ref6">6</xref>], 2021</td>
                <td>ERIN<sup>c</sup></td>
                <td>15 undergraduate students</td>
                <td>Help users with finding resources about sensitive issues</td>
                <td>Survey results: user ratings for experience</td>
                <td>Text</td>
                <td>Rule-based architecture</td>
              </tr>
              <tr valign="top">
                <td>Trappey et al [<xref ref-type="bibr" rid="ref23">23</xref>], 2022</td>
                <td>Virtual reality empathy-centric counseling CA</td>
                <td>120 university students</td>
                <td>Provide complementary support for students who were troubled</td>
                <td>Survey results: user ratings of stress levels, life impact, and psychological sensitivity</td>
                <td>Voice and text</td>
                <td>Transformer architecture</td>
              </tr>
              <tr valign="top">
                <td>Ghandeharioun et al [<xref ref-type="bibr" rid="ref24">24</xref>], 2019</td>
                <td>EMMA<sup>d</sup></td>
                <td>39 participants</td>
                <td>Delivery of just-in-time MH<sup>e</sup> interventions</td>
                <td>Survey results: user ratings of preference; behavioral metrics: user engagement</td>
                <td>Text</td>
                <td>Hybrid architecture</td>
              </tr>
              <tr valign="top">
                <td>Meng and Dai [<xref ref-type="bibr" rid="ref25">25</xref>], 2021</td>
                <td>AI CA</td>
                <td>278 participants from Midwestern University</td>
                <td>Check whether the CA’s emotional support was effective in reducing people’s stress and worry</td>
                <td>Survey results: user ratings of stress, worry, and perceived support</td>
                <td>Text</td>
                <td>Transformer architecture</td>
              </tr>
              <tr valign="top">
                <td>Goel et al [<xref ref-type="bibr" rid="ref26">26</xref>], 2021</td>
                <td>Empathic CA with an attention mechanism</td>
                <td>Trained with the Facebook AI Empathic Dialogue data set</td>
                <td>Support users express their feelings and anxious thoughts</td>
                <td>None</td>
                <td>Text</td>
                <td>Neural network architecture</td>
              </tr>
              <tr valign="top">
                <td>Adikari et al [<xref ref-type="bibr" rid="ref27">27</xref>], 2022</td>
                <td>Empathic CA</td>
                <td>Data set from Cancer Chat Canada</td>
                <td>Provide empathic patient-centered MH care</td>
                <td>Behavioral metrics for user engagement</td>
                <td>Text</td>
                <td>Hybrid architecture</td>
              </tr>
              <tr valign="top">
                <td>Inkster et al [<xref ref-type="bibr" rid="ref4">4</xref>], 2018</td>
                <td>Wysa</td>
                <td>129 users with self-reported symptoms of depression</td>
                <td>Evaluation of the effectiveness and engagement levels of Wysa</td>
                <td>Survey results for symptom assessment</td>
                <td>Text</td>
                <td>Unspecified ML<sup>f</sup> architecture</td>
              </tr>
              <tr valign="top">
                <td>Beredo and Ong [<xref ref-type="bibr" rid="ref28">28</xref>], 2022</td>
                <td>Vhope</td>
                <td>Senior high school and college students (aged 17-20 years)</td>
                <td>Help the students maintain their well-being</td>
                <td>Response ratings provided by experts</td>
                <td>Text</td>
                <td>Hybrid architecture</td>
              </tr>
              <tr valign="top">
                <td>Rathnayaka et al [<xref ref-type="bibr" rid="ref29">29</xref>], 2022</td>
                <td>Bunji</td>
                <td>Australian mobile users on Google Play Store</td>
                <td>Remote health monitoring</td>
                <td>Survey results for symptom and mood assessment</td>
                <td>Text</td>
                <td>Unspecified ML architecture</td>
              </tr>
              <tr valign="top">
                <td>Morris et al [<xref ref-type="bibr" rid="ref30">30</xref>], 2018</td>
                <td>Koko</td>
                <td>37,169 individuals who signed up for the Koko platform</td>
                <td>A corpus-based approach to simulate expressed empathy</td>
                <td>Response ratings provided by users</td>
                <td>Text</td>
                <td>Hybrid architecture</td>
              </tr>
              <tr valign="top">
                <td>Ghandeharioun et al [<xref ref-type="bibr" rid="ref31">31</xref>], 2019</td>
                <td>A behavioral change CA</td>
                <td>39 participants (n=7, 18% were female, and n=32, 82% were male)</td>
                <td>Conduct experience sampling</td>
                <td>Survey results: user ratings of likability and CA intelligence</td>
                <td>Text</td>
                <td>Rule-based architecture</td>
              </tr>
              <tr valign="top">
                <td>Saha et al [<xref ref-type="bibr" rid="ref32">32</xref>], 2022</td>
                <td>Empathic CA</td>
                <td>Data set: conversations between the support seekers who were depressed</td>
                <td>Generate empathic and motivational responses</td>
                <td>Response ratings by users for fluency, adaptability, and motivation</td>
                <td>Text</td>
                <td>Transformer architecture</td>
              </tr>
              <tr valign="top">
                <td>Agnihotri et al [<xref ref-type="bibr" rid="ref33">33</xref>], 2021</td>
                <td>Topic-driven and affective CA</td>
                <td>Data set: “ScenarioSA” with affective state labels</td>
                <td>Tackle the emotional and contextual relevance for mental well-being</td>
                <td>Response ratings for emotional relevance</td>
                <td>Text</td>
                <td>Transformer architecture</td>
              </tr>
              <tr valign="top">
                <td>Rani et al [<xref ref-type="bibr" rid="ref5">5</xref>], 2023</td>
                <td>Saarthi</td>
                <td>None</td>
                <td>None</td>
                <td>None</td>
                <td>Text</td>
                <td>Unspecified ML architecture</td>
              </tr>
              <tr valign="top">
                <td>Alazraki et al [<xref ref-type="bibr" rid="ref34">34</xref>], 2021</td>
                <td>An empathic AI coach</td>
                <td>23 participants recruited through crowd working websites</td>
                <td>Achieve a high level of engagement during web-based therapy sessions</td>
                <td>Survey results: user ratings of empathy and expert ratings of fluency</td>
                <td>Text</td>
                <td>Hybrid architecture</td>
              </tr>
              <tr valign="top">
                <td>Gundavarapu et al [<xref ref-type="bibr" rid="ref35">35</xref>], 2022</td>
                <td>A CA companion</td>
                <td>Data set: created using sources such as Wikipedia</td>
                <td>Provide emotional support, without judgment</td>
                <td>None</td>
                <td>Text</td>
                <td>Neural network architecture</td>
              </tr>
              <tr valign="top">
                <td>Mishra et al [<xref ref-type="bibr" rid="ref36">36</xref>], 2023</td>
                <td>Counseling CA</td>
                <td>A novel conversational data set</td>
                <td>Provide MH and legal counseling</td>
                <td>Survey results: user ratings of empathy</td>
                <td>Text</td>
                <td>Transformer architecture</td>
              </tr>
            </tbody>
          </table>
          <table-wrap-foot>
            <fn id="table1fn1">
              <p><sup>a</sup>CA: conversational agent.</p>
            </fn>
            <fn id="table1fn2">
              <p><sup>b</sup>AI: artificial intelligence.</p>
            </fn>
            <fn id="table1fn3">
              <p><sup>c</sup>ERIN: Empathetic Research IoT Network.</p>
            </fn>
            <fn id="table1fn4">
              <p><sup>d</sup>EMMA: Emotion-aware mHealth agent.</p>
            </fn>
            <fn id="table1fn5">
              <p><sup>e</sup>MH: mental health.</p>
            </fn>
            <fn id="table1fn6">
              <p><sup>f</sup>ML: machine learning.</p>
            </fn>
          </table-wrap-foot>
        </table-wrap>
        <p>Of the 19 studies, 6 (32%) were conducted in the United States and 6 (32%) in India. In addition, 1 (5%) study each from Australia, Canada, China, the Philippines, Poland, Switzerland, and the United Kingdom were also included. The year of publication is summarized in <xref ref-type="supplementary-material" rid="app3">Multimedia Appendix 3</xref>, indicating a sharp rise in the number of publications since 2022. Most studies, 14 (74%) out of 19, described both design and human evaluations. The types of study designs among the 19 studies included are 9 (47%) cross-sectional studies, 5 (26%) randomized controlled trials (RCTs), 4 (21%) quasi-experimental designs, and 1 (5%) qualitative study. Only 5 (26%) of the 19 studies referred to an explicit definition of empathy, as summarized in <xref ref-type="boxed-text" rid="box1">Textbox 1</xref>.</p>
        <boxed-text id="box1" position="float">
          <title>Definitions of empathy.</title>
          <p>
            <bold>Studies and definition of empathy</bold>
          </p>
          <list list-type="bullet">
            <list-item>
              <p>Jiang et al [<xref ref-type="bibr" rid="ref21">21</xref>], 2022</p>
              <list>
                <list-item>
                  <p>Empathy processing is a situation-specific, cognitive-affective state or process with the projection of oneself into another’s feelings, actions, and experiences.</p>
                </list-item>
              </list>
            </list-item>
            <list-item>
              <p>Trappey et al [<xref ref-type="bibr" rid="ref23">23</xref>], 2022</p>
              <list>
                <list-item>
                  <p>Roger’s [<xref ref-type="bibr" rid="ref39">39</xref>] definition of empathy:</p>
                  <list list-type="bullet">
                    <list-item>
                      <p>Level 1: responding to an individual’s explicitly expressed meaning and feelings with a simple repetition of basic understanding.</p>
                    </list-item>
                    <list-item>
                      <p>Level 2: responding to the implicit, half-expressed, or implied feelings of the person with corresponding emotional words to acknowledge them and bring their true feelings to the surface.</p>
                    </list-item>
                    <list-item>
                      <p>Level 3: recognizing the individual’s confusing and contradictory feelings that subconsciously obscure what they really care about, capturing the core of the emotion, and then responding to the patient’s desire with affirmations.</p>
                    </list-item>
                    <list-item>
                      <p>Level 4: when the person is suppressing their feelings or not expressing their feelings in the conversation, guessing their intentions from what they are describing, capturing the core of the emotion, and responding to it directly or indirectly in a way that is acceptable to the person.</p>
                    </list-item>
                  </list>
                </list-item>
              </list>
            </list-item>
            <list-item>
              <p>Rathnayaka et al [<xref ref-type="bibr" rid="ref29">29</xref>], 2022</p>
              <list>
                <list-item>
                  <p>Empathic engagement means, “making the impression of a credible and trustworthy conversation partner that can hear you out and offer a detached point of view on things.”</p>
                </list-item>
              </list>
            </list-item>
            <list-item>
              <p>Saha et al [<xref ref-type="bibr" rid="ref32">32</xref>], 2022</p>
              <list>
                <list-item>
                  <p>Empathy or empathic interactions refer to the ability to feel the emotions and experiences of others [<xref ref-type="bibr" rid="ref40">40</xref>].</p>
                </list-item>
              </list>
            </list-item>
            <list-item>
              <p>Alazraki et al [<xref ref-type="bibr" rid="ref34">34</xref>], 2021</p>
              <list>
                <list-item>
                  <p>Definition of empathy by Barrett-Lennard [<xref ref-type="bibr" rid="ref41">41</xref>]:</p>
                  <list list-type="bullet">
                    <list-item>
                      <p>First phase: where the listener sympathizes and resonates with what is being expressed by the speaker.</p>
                    </list-item>
                    <list-item>
                      <p>Second phase: where the listener compassionately responds to the speaker.
                                Third phase: where the speaker assimilates the listener’s response.</p>
                    </list-item>
                  </list>
                </list-item>
              </list>
            </list-item>
          </list>
        </boxed-text>
        <p>Keywords used to identify a CA varied across studies from “chatbot” (9/19, 47%) to “conversational agent” (6/19, 32%) to “dialog system” (2/19, 11%) to “virtual assistant” (1/19, 5%) to “conversational AI agent” (1/19, 5%). The mode of interaction chosen by most of the CA designs, 17 (89%) out of 19, was text (eg, live chat, symptom checker, and text-based counseling), with voice interactions being used in interactive avatar and counseling roles in 2 (11%) studies.</p>
        <p>In the <italic>Technical Design of the CAs</italic> section, we consider the technical designs used for these CAs and their performance in detecting and responding to user emotions before discussing how human-user evaluations were conducted and the conclusions reached from these evaluations.</p>
      </sec>
      <sec>
        <title>Technical Design of the CAs</title>
        <p>The types of CA architectures (or engines) considered by the authors included a mix of recent technologies, as summarized in <xref rid="figure2" ref-type="fig">Figure 2</xref>, with ML-based architectures used in 12 (63%) out of 19 cases. The transformer-based engine, which learns meaning from context, was used in 7 of the 19 (37%) studies, sometimes in the form of a large language model (LLM). A minority of the papers, 3 (16%) out of 19, did not specify the type of engine used within the design. Hybrid or ensemble models use several models in parallel to improve the accuracy of the overall CA design. A more detailed breakdown of the CA engine types with explanations is shown in <xref ref-type="supplementary-material" rid="app4">Multimedia Appendix 4</xref>. Figures S1 and S2 in <xref ref-type="supplementary-material" rid="app4">Multimedia Appendix 4</xref> also illustrate how a single engine and a hybrid engine work with user input to provide an empathic response.</p>
        <fig id="figure2" position="float">
          <label>Figure 2</label>
          <caption>
            <p>Types of conversational agent (CA) architectures. ML: machine learning.</p>
          </caption>
          <graphic xlink:href="mental_v11i1e58974_fig2.png" alt-version="no" mimetype="image" position="float" xlink:type="simple"/>
        </fig>
        <p>Transformer-based engines included Bidirectional Encoder Representations Transformer (BERT), Sentence-BERT, Robustly Optimized-BERT, Generative Pre-trained Transformer 2, and sequence-2-sequence models. Other neural network architecture–based CA designs were incorporated in 2 (11%) of the 19 papers [<xref ref-type="bibr" rid="ref26">26</xref>,<xref ref-type="bibr" rid="ref35">35</xref>].</p>
        <p>Of the 19 publications, 5 (26%) considered hybrid models. Of these hybrid models, 2 applied a ML model to capture user emotion and then applied a rule-based algorithm to generate appropriate responses in dialogue management [<xref ref-type="bibr" rid="ref24">24</xref>-<xref ref-type="bibr" rid="ref27">27</xref>]. For example, EMMA gathered mobile sensor data to infer user mood and then assigned users to appropriate wellness interventions [<xref ref-type="bibr" rid="ref24">24</xref>]. Once assigned, the CA then responded with emotionally expressive responses selected at random from a set of prescripted phrases using a rule-based approach [<xref ref-type="bibr" rid="ref27">27</xref>]. In another example, VHope, an internet-based therapist, used a hybrid model containing a retrieval model that deciphered user input combined with a generative model to elicit empathic responses [<xref ref-type="bibr" rid="ref28">28</xref>].</p>
        <p>Among the 19 papers, the 3 (16%) papers using unspecified architectures commenced with natural language processing (NLP) before using various ML approaches. In one example, continuous emotional support via remote MH care monitoring and personalized assistance was provided [<xref ref-type="bibr" rid="ref29">29</xref>]. MH monitoring was performed by scheduling activities that were meaningful to each user, sending out reminders as encouragement, and forwarding satisfaction surveys to receive feedback.</p>
        <p>Overall, 2 (11%) of the 19 publications implemented CA design approaches based on rule-based NLP architectures. For example, a mobile phone–based CA measured the level of emotion in user input and then selected an appropriate empathic response from a set of predefined scripts using a rule-based decision tree [<xref ref-type="bibr" rid="ref31">31</xref>]. In the <italic>Summary of the Results of the Assessment of the Technical Design of CAs in Terms of Classification Accuracy</italic> section, we will discuss the technical performance of the CAs reviewed.</p>
      </sec>
      <sec>
        <title>Summary of the Results of the Assessment of the Technical Design of CAs in Terms of Classification Accuracy</title>
        <p>The accuracy of the designs in detecting and responding to user emotions appropriately is summarized in <xref ref-type="table" rid="table2">Table 2</xref>. Technical evaluations of the CA designs usually involved comparisons with a “gold standard,” using data not previously used for training the CA.</p>
        <table-wrap position="float" id="table2">
          <label>Table 2</label>
          <caption>
            <p>Measures used for evaluating the technical performance of CA<sup>a</sup> designs.</p>
          </caption>
          <table width="1000" cellpadding="5" cellspacing="0" border="1" rules="groups" frame="hsides">
            <col width="300"/>
            <col width="250"/>
            <col width="450"/>
            <thead>
              <tr valign="top">
                <td>Type of CA assessment</td>
                <td>Assessment of user emotions or CA responses</td>
                <td>Accuracy measure</td>
              </tr>
            </thead>
            <tbody>
              <tr valign="top">
                <td>Classification of sentiment and issues</td>
                <td>User emotions</td>
                <td>
                  <list list-type="bullet">
                    <list-item>
                      <p>Mathews correlation coefficient=0.857 [<xref ref-type="bibr" rid="ref23">23</xref>]</p>
                    </list-item>
                  </list>
                </td>
              </tr>
              <tr valign="top">
                <td>Classification of valence and arousal</td>
                <td>User emotions</td>
                <td>
                  <list list-type="bullet">
                    <list-item>
                      <p>Accuracy of valence=80.4% [<xref ref-type="bibr" rid="ref24">24</xref>]</p>
                    </list-item>
                    <list-item>
                      <p>Accuracy of arousal=50.4% [<xref ref-type="bibr" rid="ref24">24</xref>]</p>
                    </list-item>
                  </list>
                </td>
              </tr>
              <tr valign="top">
                <td>Classification of recommended resources (for patients)</td>
                <td>User emotions</td>
                <td>
                  <list list-type="bullet">
                    <list-item>
                      <p>F1-score=0.87 [<xref ref-type="bibr" rid="ref27">27</xref>]</p>
                    </list-item>
                  </list>
                </td>
              </tr>
              <tr valign="top">
                <td>Classification of objections during conversations</td>
                <td>User emotions</td>
                <td>
                  <list list-type="bullet">
                    <list-item>
                      <p>Accuracy=99.2% [<xref ref-type="bibr" rid="ref4">4</xref>]</p>
                    </list-item>
                    <list-item>
                      <p>Specificity=99.7% [<xref ref-type="bibr" rid="ref4">4</xref>]</p>
                    </list-item>
                    <list-item>
                      <p>Precision=74.7% [<xref ref-type="bibr" rid="ref4">4</xref>]</p>
                    </list-item>
                    <list-item>
                      <p>Recall=62.1% [<xref ref-type="bibr" rid="ref4">4</xref>]</p>
                    </list-item>
                  </list>
                </td>
              </tr>
              <tr valign="top">
                <td>Performance of the topic classifier</td>
                <td>User emotions</td>
                <td>
                  <list list-type="bullet">
                    <list-item>
                      <p>Accuracy=95% [<xref ref-type="bibr" rid="ref33">33</xref>]</p>
                    </list-item>
                    <list-item>
                      <p>Precision=0.954 [<xref ref-type="bibr" rid="ref33">33</xref>]</p>
                    </list-item>
                    <list-item>
                      <p>Recall=0.947 [<xref ref-type="bibr" rid="ref33">33</xref>]</p>
                    </list-item>
                    <list-item>
                      <p>F1-score=0.95 [<xref ref-type="bibr" rid="ref33">33</xref>]</p>
                    </list-item>
                  </list>
                </td>
              </tr>
              <tr valign="top">
                <td>Classification for empathy function</td>
                <td>User emotions</td>
                <td>
                  <list list-type="bullet">
                    <list-item>
                      <p>Accuracy=80.18% [<xref ref-type="bibr" rid="ref34">34</xref>]</p>
                    </list-item>
                    <list-item>
                      <p>F1-score=80.66% [<xref ref-type="bibr" rid="ref34">34</xref>]</p>
                    </list-item>
                    <list-item>
                      <p>W-ACC<sup>b</sup>=0.977 [<xref ref-type="bibr" rid="ref36">36</xref>]</p>
                    </list-item>
                    <list-item>
                      <p>Macro F1-score=0.972 [<xref ref-type="bibr" rid="ref36">36</xref>]</p>
                    </list-item>
                  </list>
                </td>
              </tr>
              <tr valign="top">
                <td>Prediction of valence and arousal</td>
                <td>User emotions</td>
                <td>
                  <list list-type="bullet">
                    <list-item>
                      <p>Accuracy of valence=82.2% [<xref ref-type="bibr" rid="ref24">24</xref>]</p>
                    </list-item>
                    <list-item>
                      <p>Accuracy of arousal=65.7% [<xref ref-type="bibr" rid="ref24">24</xref>]</p>
                    </list-item>
                  </list>
                </td>
              </tr>
              <tr valign="top">
                <td>Accuracy of the response generation</td>
                <td>CA responses</td>
                <td>
                  <list list-type="bullet">
                    <list-item>
                      <p>BLEU<sup>c</sup> score=0.126 [<xref ref-type="bibr" rid="ref26">26</xref>]</p>
                    </list-item>
                    <list-item>
                      <p>BLEU-1 score (focused on a single word)=0.161 [<xref ref-type="bibr" rid="ref32">32</xref>]</p>
                    </list-item>
                    <list-item>
                      <p>Perplexity score=50.90 [<xref ref-type="bibr" rid="ref32">32</xref>]</p>
                    </list-item>
                    <list-item>
                      <p>ROUGE-L<sup>d</sup> score=0.124 [<xref ref-type="bibr" rid="ref32">32</xref>]</p>
                    </list-item>
                    <list-item>
                      <p>Embedding-based metrics:</p>
                      <list list-type="bullet">
                        <list-item>
                          <p>Average=0.733 [<xref ref-type="bibr" rid="ref32">32</xref>]</p>
                        </list-item>
                        <list-item>
                          <p>Extrema=0.377 [<xref ref-type="bibr" rid="ref32">32</xref>]</p>
                        </list-item>
                        <list-item>
                          <p>Greedy=0.478 [<xref ref-type="bibr" rid="ref32">32</xref>]</p>
                        </list-item>
                      </list>
                    </list-item>
                  </list>
                </td>
              </tr>
              <tr valign="top">
                <td>Emotion prediction</td>
                <td>User emotions</td>
                <td>
                  <list list-type="bullet">
                    <list-item>
                      <p>Accuracy</p>
                      <list list-type="bullet">
                        <list-item>
                          <p>Correctly predict the next emotion as positive or negative=79% [<xref ref-type="bibr" rid="ref27">27</xref>]</p>
                        </list-item>
                        <list-item>
                          <p>Proportion of correct emotion out of all emotions predicted=63% [<xref ref-type="bibr" rid="ref27">27</xref>]</p>
                        </list-item>
                      </list>
                    </list-item>
                  </list>
                </td>
              </tr>
              <tr valign="top">
                <td>Performance of the language model</td>
                <td>CA responses</td>
                <td>
                  <list list-type="bullet">
                    <list-item>
                      <p>Perplexity score=9.977 [<xref ref-type="bibr" rid="ref28">28</xref>]</p>
                    </list-item>
                    <list-item>
                      <p>Perplexity score=1.91 [<xref ref-type="bibr" rid="ref23">23</xref>]</p>
                    </list-item>
                    <list-item>
                      <p>Response length=18.71 [<xref ref-type="bibr" rid="ref23">23</xref>]</p>
                    </list-item>
                  </list>
                </td>
              </tr>
              <tr valign="top">
                <td>Emotion recognition</td>
                <td>User emotions</td>
                <td>
                  <list list-type="bullet">
                    <list-item>
                      <p>Accuracy=94.96% [<xref ref-type="bibr" rid="ref34">34</xref>]</p>
                    </list-item>
                    <list-item>
                      <p>F1-score=95.10% [<xref ref-type="bibr" rid="ref34">34</xref>]</p>
                    </list-item>
                  </list>
                </td>
              </tr>
            </tbody>
          </table>
          <table-wrap-foot>
            <fn id="table2fn1">
              <p><sup>a</sup>CA: conversational agent.</p>
            </fn>
            <fn id="table2fn2">
              <p><sup>b</sup>W-ACC: weighted accuracy.</p>
            </fn>
            <fn id="table2fn3">
              <p><sup>c</sup>BLEU: Bilingual Evaluation Understudy.</p>
            </fn>
            <fn id="table2fn4">
              <p><sup>d</sup>ROUGE-L: Recall Oriented Understudy for Gisting Evaluation–Longest Common Sequence.</p>
            </fn>
          </table-wrap-foot>
        </table-wrap>
        <p>A technical evaluation of empathic CA performance was conducted in 17 (89%) of the 19 papers reviewed; however, only 10 (53%) papers reported these results. These studies conducted comprehensive assessments where technical performance was measured in terms of recognition, classification, prediction, and response generation abilities during interactions with end users. The assessments were centered around the ability of the CA to discern user emotions correctly and to respond appropriately. Of the 19 papers, 4 (21%) focused on the CA responses during the technical assessments, while the rest of the studies (n=15, 79%) considered user emotions. A variety of measures were used for each such assessment, highlighting the diversity in evaluation methodologies across studies. These metrics are categorized in detail under the type of CA performance in <xref ref-type="supplementary-material" rid="app5">Multimedia Appendix 5</xref> [<xref ref-type="bibr" rid="ref4">4</xref>-<xref ref-type="bibr" rid="ref6">6</xref>,<xref ref-type="bibr" rid="ref23">23</xref>-<xref ref-type="bibr" rid="ref34">34</xref>,<xref ref-type="bibr" rid="ref36">36</xref>].</p>
        <p>In general, the performances of the CA designs were satisfactory. The highest classification accuracy for user emotions was reported by ML-based CAs. In one of these studies, a Robustly Optimized-BERT transformer model, which was built integrating 3 classifiers for politeness, counseling strategy, and empathic feedback, achieved good results overall. This empathy classifier achieved excellent performance with a weighted accuracy score of 0.977 and an <italic>F</italic><sub>1</sub>-score of 0.972 [<xref ref-type="bibr" rid="ref36">36</xref>]. In a second study, a topic-driven classification model used a Generative Pre-trained Transformer 2 model for generating controlled responses, and the classification model accomplished relatively high scores of accuracy (95%), precision (0.954), and recall (0.947) and an <italic>F</italic><sub>1</sub>-score of 0.95 [<xref ref-type="bibr" rid="ref33">33</xref>].</p>
        <p>However, high accuracy and a more nuanced response generation were consistently apparent in all the CAs using hybrid architectures [<xref ref-type="bibr" rid="ref24">24</xref>,<xref ref-type="bibr" rid="ref27">27</xref>,<xref ref-type="bibr" rid="ref28">28</xref>,<xref ref-type="bibr" rid="ref30">30</xref>,<xref ref-type="bibr" rid="ref34">34</xref>], suggesting that hybrid models lead to enhanced performance in tasks requiring complex understanding of user emotions and the generation of contextual responses.</p>
      </sec>
      <sec>
        <title>Human Evaluation of CAs</title>
        <p>Most of the reviewed studies, 16 (84%) out of 19, conducted a human evaluation of the implemented CA designs. Acceptability by end users was evaluated in terms of user experience, satisfaction, and levels of engagement. A detailed summary of the human evaluations of these designs is presented in <xref ref-type="supplementary-material" rid="app5">Multimedia Appendix 5</xref>.</p>
        <p>The human evaluation was performed by only CA users in most cases (13/16, 81%), while experts in the field of MH contributed to the process of assessing the CA in the remaining studies (3/16, 19%). <xref ref-type="table" rid="table3">Table 3</xref> summarizes the empathy measures used in these papers.</p>
        <table-wrap position="float" id="table3">
          <label>Table 3</label>
          <caption>
            <p>Measurement of empathy in CAs<sup>a</sup>.</p>
          </caption>
          <table width="1000" cellpadding="5" cellspacing="0" border="1" rules="groups" frame="hsides">
            <col width="130"/>
            <col width="260"/>
            <col width="130"/>
            <col width="220"/>
            <col width="260"/>
            <thead>
              <tr valign="top">
                <td>Study and year</td>
                <td>The method of empathy measurement</td>
                <td>How was empathy measured?</td>
                <td>Who did the evaluation?</td>
                <td>Evaluation results</td>
              </tr>
            </thead>
            <tbody>
              <tr valign="top">
                <td>Jiang et al [<xref ref-type="bibr" rid="ref21">21</xref>], 2022</td>
                <td>
                  <list list-type="bullet">
                    <list-item>
                      <p>Self-reports:</p>
                      <list list-type="bullet">
                        <list-item>
                          <p>In-depth interview responses</p>
                        </list-item>
                        <list-item>
                          <p>Multiple response ratings</p>
                        </list-item>
                      </list>
                    </list-item>
                  </list>
                </td>
                <td>Using the RoPE<sup>b</sup> scale (binary responses) and QCAE<sup>c</sup></td>
                <td>
                  <list list-type="bullet">
                    <list-item>
                      <p>Replika users provided the empathy ratings</p>
                    </list-item>
                  </list>
                </td>
                <td>
                  <list list-type="bullet">
                    <list-item>
                      <p>Perceived cognitive empathy was higher than perceived affective empathy</p>
                    </list-item>
                  </list>
                </td>
              </tr>
              <tr valign="top">
                <td>Beredo and Ong [<xref ref-type="bibr" rid="ref28">28</xref>], 2022</td>
                <td>
                  <list list-type="bullet">
                    <list-item>
                      <p>Self-reports:</p>
                      <list list-type="bullet">
                        <list-item>
                          <p>Response ratings</p>
                        </list-item>
                      </list>
                    </list-item>
                  </list>
                </td>
                <td>Affect criterion or empathy was measured using a binary scale of 0 (no) to 1 (yes)</td>
                <td>
                  <list list-type="bullet">
                    <list-item>
                      <p>Evaluated by 3 experts who studied and practiced psychology</p>
                    </list-item>
                  </list>
                </td>
                <td>
                  <list list-type="bullet">
                    <list-item>
                      <p>Responses were rated 79% empathic</p>
                    </list-item>
                  </list>
                </td>
              </tr>
              <tr valign="top">
                <td>Alazraki et al [<xref ref-type="bibr" rid="ref34">34</xref>], 2021</td>
                <td>
                  <list list-type="bullet">
                    <list-item>
                      <p>Self-reports:</p>
                      <list list-type="bullet">
                        <list-item>
                          <p>Multiple response ratings</p>
                        </list-item>
                      </list>
                    </list-item>
                  </list>
                </td>
                <td>Multiple ratings to evaluate the perceived level of empathy, with ratings ranging from strongly disagree to strongly agree on a 5-point Likert scale</td>
                <td>
                  <list list-type="bullet">
                    <list-item>
                      <p>Evaluated by users</p>
                    </list-item>
                    <list-item>
                      <p>2 separate clinicians specialized in MH<sup>d</sup> also evaluated the chatbot personas</p>
                    </list-item>
                  </list>
                </td>
                <td>
                  <list list-type="bullet">
                    <list-item>
                      <p>When interacting with the Kai persona, 75% of users agreed that the bot was empathic</p>
                    </list-item>
                    <list-item>
                      <p>Interaction with other study personas achieved a 56% empathic rating</p>
                    </list-item>
                  </list>
                </td>
              </tr>
              <tr valign="top">
                <td>Mishra et al [<xref ref-type="bibr" rid="ref36">36</xref>], 2023</td>
                <td>
                  <list list-type="bullet">
                    <list-item>
                      <p>Self-reports:</p>
                      <list list-type="bullet">
                        <list-item>
                          <p>Response ratings</p>
                        </list-item>
                      </list>
                    </list-item>
                  </list>
                </td>
                <td>A single 5-point Likert scale</td>
                <td>
                  <list list-type="bullet">
                    <list-item>
                      <p>6 evaluators rated each dialogue interaction for empathy</p>
                    </list-item>
                    <list-item>
                      <p>Empathy ratings by evaluators cross-validated for quality by government-run institutions</p>
                    </list-item>
                  </list>
                </td>
                <td>
                  <list list-type="bullet">
                    <list-item>
                      <p>Average empathy rating=57%</p>
                    </list-item>
                  </list>
                </td>
              </tr>
              <tr valign="top">
                <td>Agnihotri et al [<xref ref-type="bibr" rid="ref33">33</xref>], 2021</td>
                <td>
                  <list list-type="bullet">
                    <list-item>
                      <p>Self-reports:</p>
                      <list list-type="bullet">
                        <list-item>
                          <p>Response ratings</p>
                        </list-item>
                      </list>
                    </list-item>
                  </list>
                </td>
                <td>Emotional relevance is rated using a single 5-point Likert scale</td>
                <td>
                  <list list-type="bullet">
                    <list-item>
                      <p>Evaluated by 3 human annotators—male nonnative English speakers from a technical university with an average age of 21 years</p>
                    </list-item>
                  </list>
                </td>
                <td>When an empathic response generator was used, emotional relevance=61.4%<break/><list list-type="bullet"><list-item><p>When a topic classifier was added, emotional relevance=43%</p></list-item></list></td>
              </tr>
            </tbody>
          </table>
          <table-wrap-foot>
            <fn id="table3fn1">
              <p><sup>a</sup>CA: conversational agent.</p>
            </fn>
            <fn id="table3fn2">
              <p><sup>b</sup>RoPE: Robot’s Perceived Empathy.</p>
            </fn>
            <fn id="table3fn3">
              <p><sup>c</sup>QCAE: Questionnaire of Cognitive and Affective Empathy.</p>
            </fn>
            <fn id="table3fn4">
              <p><sup>d</sup>MH: mental health.</p>
            </fn>
          </table-wrap-foot>
        </table-wrap>
        <p>Alazraki et al [<xref ref-type="bibr" rid="ref34">34</xref>] conducted a cross-sectional study with 23 volunteers and 2 clinicians who engaged with a web-based chatbot platform using 4 prescripted conversations of different CA personas. An anonymous web-based questionnaire collected participant feedback regarding the level of empathy displayed by the chatbot, engagement levels, and the ability of the chatbot to identify emotions in the participant. The survey results revealed that 75% of users agreed that the CA persona Kai was empathic, 63% found it engaging, and 75% rated it as useful. In contrast, Beredo and Ong [<xref ref-type="bibr" rid="ref28">28</xref>] asked 3 psychologists to provide feedback on chatbot user logs. Empathy was measured using the affect criterion, a measure of the ability of the CA to read and respond to users with empathy, along with performance and humanlike characteristics. On the basis of expert feedback, 67% of the CA responses were relevant, 78% seemed human, and 70% were empathic.</p>
        <p>In an RCT, a group of 39 participants were randomly allocated to a treatment group interacting with the emotion-aware chatbot EMMA, while a control group (n=39) was assigned to an emotionally nonexpressive chatbot, with 2 weeks of monitoring in each case [<xref ref-type="bibr" rid="ref24">24</xref>]. The participants engaging with EMMA showed higher frequency of interactions and responded quicker than the control group. The feedback of the users was useful in understanding how empathy was perceived during the study.</p>
        <p>The only qualitative experimental study involved an AI-based chatbot, Replika, designed to improve resilience and user well-being [<xref ref-type="bibr" rid="ref21">21</xref>]. The author followed an ethnographic approach for their study of empathy, asking users to download the Replika application and write down reflective notes on their conversations with Replika. The results of this study expand the empathy theories within human conversations to human-AI interactions through variations in cognitive empathy, affective empathy, and empathic responses. A list of technical terms used in the paper is further explained in <xref ref-type="supplementary-material" rid="app6">Multimedia Appendix 6</xref>.</p>
      </sec>
      <sec>
        <title>Risk of Bias and Quality Assessment Results</title>
        <p>The included RCTs showed a low risk of bias on the revised Cochrane risk-of-bias tool. Of the 14 nonrandomized studies included in the review, all showed a moderate to high risk of bias. A total of 5 (36%) studies [<xref ref-type="bibr" rid="ref27">27</xref>,<xref ref-type="bibr" rid="ref32">32</xref>-<xref ref-type="bibr" rid="ref34">34</xref>,<xref ref-type="bibr" rid="ref36">36</xref>] were moderately biased, and 1 (7%) study [<xref ref-type="bibr" rid="ref28">28</xref>] was seriously biased according to the risk of bias in nonrandomized studies of interventions tool. The Joanna Briggs Institute quality assessment results were generally low when only the design component of the studies was assessed, with 32% (6/19) of the papers receiving a score of 0. However, an overall moderate quality was seen in publications when both the design and implementation stages were appraised. <xref ref-type="supplementary-material" rid="app7">Multimedia Appendix 7</xref> [<xref ref-type="bibr" rid="ref4">4</xref>-<xref ref-type="bibr" rid="ref6">6</xref>,<xref ref-type="bibr" rid="ref21">21</xref>-<xref ref-type="bibr" rid="ref36">36</xref>] shows the quality assessment results.</p>
      </sec>
    </sec>
    <sec sec-type="discussion">
      <title>Discussion</title>
      <sec>
        <title>Principal Findings</title>
        <p>The study and use of CA technology have been the subject of extensive research across many fields, such as education, customer service, and health care. Moreover, there are reviews focusing on AI-based CAs, their effectiveness, and their impact in the realm of MH care [<xref ref-type="bibr" rid="ref17">17</xref>,<xref ref-type="bibr" rid="ref18">18</xref>,<xref ref-type="bibr" rid="ref42">42</xref>]. While these reviews offer significant insights into AI-based CA designs in MH care, the importance of empathy is not central. Although these reviews suggest the need for empathy in CA innovations in MH care, they do not consider CA designs specifically aimed at generating and evaluating empathy. To address this gap, this review compares various empathic CA designs, their effectiveness in detecting and responding to user emotions, and their acceptability to users.</p>
      </sec>
      <sec>
        <title>CA Designs</title>
        <p>This review has found that most researchers used an ML-based transformer engine for designing empathic CAs, achieving excellent classification and prediction results. Surprisingly, several researchers used rule-based architectures and retrieval engines. While lacking the sophistication of transformer-based engines in terms of comprehension, rule-based approaches were able to efficiently identify keywords and themes, ensuring that consumer needs were addressed within a limited number of categories. Rule-based systems are comparatively easy to design and implement, allowing for a trade-off between classification accuracy and economic feasibility. However, rule-based systems tend to generate more predictable, inflexible, and repetitive responses compared to advanced LLM engines and, therefore, might be more suitable for providing simple information to managers and MH care workers, rather than responding to end users requiring more nuanced responses.</p>
        <p>Hybrid architecture seems best suited to the detection of user emotion followed by the retrieval of a suitable response. Therefore, having &#62;1 model appears to facilitate a more robust model output. This is supported by the superior accuracies achieved by hybrid architectures in the classification and prediction tasks. The hybrid model of Adikari et al [<xref ref-type="bibr" rid="ref27">27</xref>] achieved the highest accuracy of 87% (<italic>F</italic><sub>1</sub>-score=0.87) in recommending a resource based on the concerns expressed by the patients. However, the highest accuracy in emotion recognition (95% accuracy in identifying sadness, anger, fear, and happiness) was obtained by Alazraki et al [<xref ref-type="bibr" rid="ref34">34</xref>]. The combined features of high accuracy and improved user experience probably make these the best performing CAs within the review.</p>
        <p>While the use of such robust LLMs has significantly improved language-based CA technology, it is important to recognize that these models are not without disadvantages [<xref ref-type="bibr" rid="ref43">43</xref>]. These models have been found to perpetuate biases with regard to gender, race, and MH conditions present in the training data [<xref ref-type="bibr" rid="ref44">44</xref>,<xref ref-type="bibr" rid="ref45">45</xref>]. Such biases can strengthen gender stereotypes and reduce response accuracy when dealing with users from diverse cultural backgrounds, potentially causing harm to users. Such issues may have serious impacts on user trust, the credibility of the empathic CA, and user well-being. Such biases can be mitigated by ensuring that the training data sets represent diverse gender categories, races, and cultural backgrounds and that advanced technical approaches are used to detect and minimize any such biases in the training data [<xref ref-type="bibr" rid="ref46">46</xref>-<xref ref-type="bibr" rid="ref48">48</xref>].</p>
        <p>Ethical and privacy concerns associated with these LLMs are critical [<xref ref-type="bibr" rid="ref49">49</xref>,<xref ref-type="bibr" rid="ref50">50</xref>]. Following ethical guidelines centered around transparency, accountability, and adherence are pivotal to user privacy, while measures to maintain data security through strict access controls and regular security checks also need to be in place. Privacy should be a core component of CA designs, with limitations placed on personal data collection whenever possible [<xref ref-type="bibr" rid="ref7">7</xref>]. These strategies are especially important for an empathic CA design dealing with users seeking MH care. Any breaches of privacy and ethical guidelines pose a high risk to user mental well-being as well as users’ trust in and acceptance of these new technologies [<xref ref-type="bibr" rid="ref51">51</xref>]. The AI safety guidelines established by the European Union provide a key foundation for the creation of secure and ethical experiences for users [<xref ref-type="bibr" rid="ref50">50</xref>].</p>
        <p>Due to the complexity of LLMs and the many parameters involved, some models can have high latency in response time, which can cause potential challenges for a real-time CA dealing with vulnerable users waiting for a response. However, the use of parallel processing, optimization techniques, and hardware that supports the requirements of these AI models has facilitated a decrease in execution times [<xref ref-type="bibr" rid="ref52">52</xref>].</p>
      </sec>
      <sec>
        <title>Human Evaluations of CAs</title>
        <p>Among the reviewed publications, human evaluation of chatbots was common. However, only 26% (5/19) of the studies used an RCT design to assess the CA platform. Random assignment to the treatment arm is known to reduce bias while improving the reliability of the experimental results. Any confounding factors are, therefore, likely to be controlled for in an RCT, making it important to overcome the practical difficulties these designs present in this context. RCTs provide the opportunity to observe user experiences with the CA designs over time. Ideally, future studies should consider RCT designs for their human evaluations, and ideally, the long-term effects of the CA can be examined over an extended timeline.</p>
        <p>Previous experiences with CAs could be an important confounding factor. On the basis of these experiences, expectations of users regarding CA performance may affect actual engagement with the CA. Previous bad experiences may make it less likely that a user will try to engage fully with a CA, resulting in a less favorable evaluation and satisfaction levels [<xref ref-type="bibr" rid="ref53">53</xref>]. Another confounding factor could be the rate at which the user likes to communicate. If the CA cannot automatically adapt its speed of response to that preferred by the user, it is likely that this will also impact evaluation results [<xref ref-type="bibr" rid="ref54">54</xref>].</p>
        <p>The human evaluations of CAs in this review focused on their ability to portray empathy, satisfy user needs, provide useful and contextually informed responses, and facilitate user engagement. Most CAs were evaluated as satisfactory by end users. However, among the 19 papers reviewed, only 5 (26%) papers provided quantitative evaluations of CA empathy, and only 5 (26%) papers provided a definition of empathy.</p>
        <p>Because empathy has been defined in numerous ways in the literature, it is important that in future studies users are given a framework that guides their perceptions of empathy. Future research on empathic CA designs would, therefore, benefit from a clear and well-established definition of empathy, such as that provided by the World Health Organization [<xref ref-type="bibr" rid="ref8">8</xref>]. Ideally, standardized scales for perceived empathy should be used to enhance the reliability, comparability, and validity of survey results. In this review, other self-report measures were used as surrogates for empathy, with considerable variation in the types of scales used. However, self-report scales are subjective and prone to bias, with different meanings based on users’ lived experiences [<xref ref-type="bibr" rid="ref55">55</xref>]. Ideally, the impact of the CA on MH outcomes should also be assessed. Only 2 (11%) of the 19 papers in this review [<xref ref-type="bibr" rid="ref4">4</xref>,<xref ref-type="bibr" rid="ref29">29</xref>] used the Patient Health Questionnaire as their measure of MH outcomes, while 2 (11%) other papers considered stress levels in their evaluation [<xref ref-type="bibr" rid="ref23">23</xref>,<xref ref-type="bibr" rid="ref25">25</xref>].</p>
        <p>Furthermore, the human evaluations were mostly conducted by study participants. Experts and professionals in the field of MH care were rarely consulted. There is a need for greater consultation with focus groups and user groups to ensure that the CA design best reflects the needs of all stakeholders [<xref ref-type="bibr" rid="ref22">22</xref>]. Future research in this area should also consider an iterative design framework, incorporating the co-design and coevaluation of prototypes involving all stakeholders [<xref ref-type="bibr" rid="ref22">22</xref>].</p>
        <p>In summary, there were deficiencies in all the human evaluations included in this review. Only 5 (26%) of the 19 papers in this review included a direct evaluation of CA empathy in the design, while the rest (n=14, 74%) were more concerned with general user satisfaction. Only 2 (40%) of 5 these studies used multiple rating scales to measure the level of empathy portrayed by a CA, and only 1 (20%) of 5 these studies [<xref ref-type="bibr" rid="ref34">34</xref>] considered evaluations by both users and clinicians. However, there were 4 studies that did consider the impact of the CA on MH outcomes.</p>
      </sec>
      <sec>
        <title>Future Opportunities</title>
        <p>A significant limitation of the CAs reviewed was the use of only textual input in all but 2 (11%) of the 19 studies where voice data were included, thus losing a valuable opportunity to leverage alternative and powerful forms of data input for evaluating empathy.. A range of vocal characteristics have been associated with the detection of suicide risk and psychological distress, which suggests that vocal characteristics might provide a natural extension for the detection of levels of empathy [<xref ref-type="bibr" rid="ref56">56</xref>,<xref ref-type="bibr" rid="ref57">57</xref>]. The omission of voice data is surprising given that empathy is communicated predominately through vocal cues. However, textual information is not without its advantages. As we have shown in this review, NLP approaches have been used to successfully detect and convey empathy by CAs. A novel approach would be to leverage both streams of information to identify vocal characteristics indicative of different levels of empathy in addition to textual cues. Characteristics of vocal and textual cues that are associated with empathy could be combined to create a CA design to attend to users of MH care facilities such as helpline services, patient triage, and emergency services [<xref ref-type="bibr" rid="ref21">21</xref>,<xref ref-type="bibr" rid="ref23">23</xref>].</p>
        <p>Creating a CA design that accurately portrays empathy and adjusts the level of empathy to match the emotional status of patients is a significant challenge. Effective vocal interaction often faces hurdles due to technical issues in voice analysis, including the smooth processing and interpretation of data. These challenges are compounded by poor audio quality [<xref ref-type="bibr" rid="ref58">58</xref>]; the presence of overlapping psychological states in users; and linguistic variability influenced by culture, age, gender, and accents [<xref ref-type="bibr" rid="ref59">59</xref>-<xref ref-type="bibr" rid="ref61">61</xref>]. The use of high-quality audio devices to capture user voice [<xref ref-type="bibr" rid="ref62">62</xref>, <xref ref-type="bibr" rid="ref63">63</xref>] and the use of training data sets reflecting diverse human demographic features are two challenges in algorithm development aiming to provide effective vocal interaction in CAs in real time.</p>
        <p>The integration of an empathic CA with voice analysis capabilities into crisis helpline services could benefit users and the service provider. Attending to callers during peak hours for the collection of demographic information, triage, and risk assessment of callers using their voice patterns are some of the possible roles that CAs could fulfill. The involvement of CAs in these capacities could help reduce caller wait times, streamline processes, and ensure 24-hour service availability while providing a nonjudgmental and sensitive interaction for users within a safe environment. Improved empathy portrayal by the CA would help enhance user engagement and CA acceptability, helping reduce the gap between the demand and supply of available crisis helpline services.</p>
      </sec>
      <sec>
        <title>Summary</title>
        <p>This review confirms that empathy is an important characteristic for CA implementation for MH care. It highlights the strengths of the ML-based architectures when it comes to CA design and provides evidence of both technical and human assessments of CA performance. The need for improvement in measures used for detecting the level of empathy exhibited by CAs is manifest. The importance of AI safety regarding ethical and privacy concerns is a neglected area and should be considered as a priority for future designs. The promise of empathic CA applications that use vocal inputs and outputs is another area warranting further research, with opportunities for crisis helpline services.</p>
      </sec>
      <sec>
        <title>Limitations of the Review</title>
        <p>The studies included in this review presented a mix of methods, which made it challenging to compare and analyze the results. This relates to the diversity in the CA designs included, along with the different data formats obtained through human evaluations, such as survey results, response ratings, and interview feedback. The methods used to assess the accuracy of the technical designs were also varied, and a lack of empathy definitions and standard measures for perceived empathy made study comparisons difficult.</p>
        <p>The quality rating of the studies emphasized the need for the complete reporting of CA designs as well as rigorous evaluation. Deficiencies in these areas meant that the quality ratings for several papers were low. Evaluation guidelines were often missing, which made it challenging to appraise the performance of these systems. Classification accuracy and the accuracy of the responses generated were assessed using a variety of methods, further complicating this comparison.</p>
      </sec>
      <sec>
        <title>Conclusions</title>
        <p>The objective of this systematic review was to identify the existing architectures of empathic CA designs and the types of CA design assessments used in MH care. A further aim was to determine how CA empathy is evaluated and to examine the limitations and future ideas for CAs in this specific context. More than half of the selected papers used the latest technologies in CA architectures, including designs developed using ML-based transformer engines (eg, LLMs). Evaluations of technical capabilities were conducted in most of the papers and demonstrated good levels of accuracy.</p>
        <p>This review suggests that a hybrid design is ideally used for the design of an empathic CA, allowing an initial assessment of user emotion before any CA response is developed. This review indicates that human feedback is required to assess the extent to which the CA is successful in demonstrating empathy. It is recommended that well-validated scales be used for this purpose. Further research on the portrayal of empathy in CAs for MH care would benefit by involving cocreation activities, explicit definitions of empathy, and effective evaluation of empathy using standardized empathy scales, as well as by using vocal features associated with empathy in addition to textual cues.</p>
        <p>Despite its limitations, this review demonstrates that it is possible to design AI-empowered CAs that evoke empathy within MH care applications, with many of these CAs being rated as satisfactory by human users. This suggests that such CAs could prove beneficial in a range of settings, such as crisis helpline services, gathering data on user characteristics and emotions, and in postvention follow-up, helping to bridge the gap between the existing supply and demand for MH services.</p>
      </sec>
    </sec>
  </body>
  <back>
    <app-group>
      <supplementary-material id="app1">
        <label>Multimedia Appendix 1</label>
        <p>Screening process and study characteristics.</p>
        <media xlink:href="mental_v11i1e58974_app1.docx" xlink:title="DOCX File , 42 KB"/>
      </supplementary-material>
      <supplementary-material id="app2">
        <label>Multimedia Appendix 2</label>
        <p>PRISMA (Preferred Reporting Items for Systematic Reviews and Meta-Analyses) checklist 2020.</p>
        <media xlink:href="mental_v11i1e58974_app2.docx" xlink:title="DOCX File , 32 KB"/>
      </supplementary-material>
      <supplementary-material id="app3">
        <label>Multimedia Appendix 3</label>
        <p>Evolution of conversational agent (year by year).</p>
        <media xlink:href="mental_v11i1e58974_app3.docx" xlink:title="DOCX File , 56 KB"/>
      </supplementary-material>
      <supplementary-material id="app4">
        <label>Multimedia Appendix 4</label>
        <p>Detailed summary of conversational agent types.</p>
        <media xlink:href="mental_v11i1e58974_app4.docx" xlink:title="DOCX File , 49 KB"/>
      </supplementary-material>
      <supplementary-material id="app5">
        <label>Multimedia Appendix 5</label>
        <p>Results of conversational agent evaluations.</p>
        <media xlink:href="mental_v11i1e58974_app5.docx" xlink:title="DOCX File , 99 KB"/>
      </supplementary-material>
      <supplementary-material id="app6">
        <label>Multimedia Appendix 6</label>
        <p>Dictionary of technical terms.</p>
        <media xlink:href="mental_v11i1e58974_app6.docx" xlink:title="DOCX File , 16 KB"/>
      </supplementary-material>
      <supplementary-material id="app7">
        <label>Multimedia Appendix 7</label>
        <p>Risk of bias and quality assessment.</p>
        <media xlink:href="mental_v11i1e58974_app7.docx" xlink:title="DOCX File , 16 KB"/>
      </supplementary-material>
    </app-group>
    <glossary>
      <title>Abbreviations</title>
      <def-list>
        <def-item>
          <term id="abb1">AI</term>
          <def>
            <p>artificial intelligence</p>
          </def>
        </def-item>
        <def-item>
          <term id="abb2">BERT</term>
          <def>
            <p>Bidirectional Encoder Representations Transformer</p>
          </def>
        </def-item>
        <def-item>
          <term id="abb3">CA</term>
          <def>
            <p>conversational agent</p>
          </def>
        </def-item>
        <def-item>
          <term id="abb4">LLM</term>
          <def>
            <p>large language model</p>
          </def>
        </def-item>
        <def-item>
          <term id="abb5">MH</term>
          <def>
            <p>mental health</p>
          </def>
        </def-item>
        <def-item>
          <term id="abb6">ML</term>
          <def>
            <p>machine learning</p>
          </def>
        </def-item>
        <def-item>
          <term id="abb7">NLP</term>
          <def>
            <p>natural language processing</p>
          </def>
        </def-item>
        <def-item>
          <term id="abb8">PRISMA</term>
          <def>
            <p>Preferred Reporting Items for Systematic Reviews and Meta-Analyses</p>
          </def>
        </def-item>
        <def-item>
          <term id="abb9">RCT</term>
          <def>
            <p>randomized controlled trial</p>
          </def>
        </def-item>
      </def-list>
    </glossary>
    <ack>
      <p>This study was funded by Swinburne University of Technology.</p>
    </ack>
    <fn-group>
      <fn fn-type="con">
        <p>RS, DM, and RI contributed to the study selection process. RS and DM conducted the quality assessment of the included studies. RS, DM, RI, PA, and NW were involved in the concept, design, revisions, and final approval of the paper.</p>
      </fn>
      <fn fn-type="conflict">
        <p>None declared.</p>
      </fn>
    </fn-group>
    <ref-list>
      <ref id="ref1">
        <label>1</label>
        <nlm-citation citation-type="web">
          <article-title>Mental health atlas 2020</article-title>
          <source>World Health Organization</source>
          <year>2021</year>
          <month>10</month>
          <day>8</day>
          <access-date>2024-08-10</access-date>
          <comment>
            <ext-link ext-link-type="uri" xlink:type="simple" xlink:href="https://www.who.int/publications/i/item/9789240036703">https://www.who.int/publications/i/item/9789240036703</ext-link>
          </comment>
        </nlm-citation>
      </ref>
      <ref id="ref2">
        <label>2</label>
        <nlm-citation citation-type="journal">
          <person-group person-group-type="author">
            <name name-style="western">
              <surname>Schick</surname>
              <given-names>A</given-names>
            </name>
            <name name-style="western">
              <surname>Feine</surname>
              <given-names>J</given-names>
            </name>
            <name name-style="western">
              <surname>Morana</surname>
              <given-names>S</given-names>
            </name>
            <name name-style="western">
              <surname>Maedche</surname>
              <given-names>A</given-names>
            </name>
            <name name-style="western">
              <surname>Reininghaus</surname>
              <given-names>U</given-names>
            </name>
          </person-group>
          <article-title>Validity of chatbot use for mental health assessment: experimental study</article-title>
          <source>JMIR Mhealth Uhealth</source>
          <year>2022</year>
          <month>10</month>
          <day>31</day>
          <volume>10</volume>
          <issue>10</issue>
          <fpage>e28082</fpage>
          <comment>
            <ext-link ext-link-type="uri" xlink:type="simple" xlink:href="https://mhealth.jmir.org/2022/10/e28082/"/>
          </comment>
          <pub-id pub-id-type="doi">10.2196/28082</pub-id>
          <pub-id pub-id-type="medline">36315228</pub-id>
          <pub-id pub-id-type="pii">v10i10e28082</pub-id>
          <pub-id pub-id-type="pmcid">PMC9664331</pub-id>
        </nlm-citation>
      </ref>
      <ref id="ref3">
        <label>3</label>
        <nlm-citation citation-type="journal">
          <person-group person-group-type="author">
            <name name-style="western">
              <surname>Fitzpatrick</surname>
              <given-names>KK</given-names>
            </name>
            <name name-style="western">
              <surname>Darcy</surname>
              <given-names>A</given-names>
            </name>
            <name name-style="western">
              <surname>Vierhile</surname>
              <given-names>M</given-names>
            </name>
          </person-group>
          <article-title>Delivering cognitive behavior therapy to young adults with symptoms of depression and anxiety using a fully automated conversational agent (Woebot): a randomized controlled trial</article-title>
          <source>JMIR Ment Health</source>
          <year>2017</year>
          <month>06</month>
          <day>06</day>
          <volume>4</volume>
          <issue>2</issue>
          <fpage>e19</fpage>
          <comment>
            <ext-link ext-link-type="uri" xlink:type="simple" xlink:href="https://mental.jmir.org/2017/2/e19/"/>
          </comment>
          <pub-id pub-id-type="doi">10.2196/mental.7785</pub-id>
          <pub-id pub-id-type="medline">28588005</pub-id>
          <pub-id pub-id-type="pii">v4i2e19</pub-id>
          <pub-id pub-id-type="pmcid">PMC5478797</pub-id>
        </nlm-citation>
      </ref>
      <ref id="ref4">
        <label>4</label>
        <nlm-citation citation-type="journal">
          <person-group person-group-type="author">
            <name name-style="western">
              <surname>Inkster</surname>
              <given-names>B</given-names>
            </name>
            <name name-style="western">
              <surname>Sarda</surname>
              <given-names>S</given-names>
            </name>
            <name name-style="western">
              <surname>Subramanian</surname>
              <given-names>V</given-names>
            </name>
          </person-group>
          <article-title>An empathy-driven, conversational artificial intelligence agent (Wysa) for digital mental well-being: real-world data evaluation mixed-methods study</article-title>
          <source>JMIR Mhealth Uhealth</source>
          <year>2018</year>
          <month>11</month>
          <day>23</day>
          <volume>6</volume>
          <issue>11</issue>
          <fpage>e12106</fpage>
          <comment>
            <ext-link ext-link-type="uri" xlink:type="simple" xlink:href="https://mhealth.jmir.org/2018/11/e12106/"/>
          </comment>
          <pub-id pub-id-type="doi">10.2196/12106</pub-id>
          <pub-id pub-id-type="medline">30470676</pub-id>
          <pub-id pub-id-type="pii">v6i11e12106</pub-id>
          <pub-id pub-id-type="pmcid">PMC6286427</pub-id>
        </nlm-citation>
      </ref>
      <ref id="ref5">
        <label>5</label>
        <nlm-citation citation-type="confproc">
          <person-group person-group-type="author">
            <name name-style="western">
              <surname>Rani</surname>
              <given-names>K</given-names>
            </name>
            <name name-style="western">
              <surname>Vishnoi</surname>
              <given-names>H</given-names>
            </name>
            <name name-style="western">
              <surname>Mishra</surname>
              <given-names>M</given-names>
            </name>
          </person-group>
          <article-title>A mental health chatbot delivering cognitive behavior therapy and remote health monitoring using NLP and AI</article-title>
          <source>Proceedings of the International Conference on Disruptive Technologies</source>
          <year>2023</year>
          <conf-name>ICDT 2023</conf-name>
          <conf-date>May 11-12, 2023</conf-date>
          <conf-loc>Greater Noida, India</conf-loc>
          <pub-id pub-id-type="doi">10.1109/icdt57929.2023.10150665</pub-id>
        </nlm-citation>
      </ref>
      <ref id="ref6">
        <label>6</label>
        <nlm-citation citation-type="confproc">
          <person-group person-group-type="author">
            <name name-style="western">
              <surname>Persons</surname>
              <given-names>B</given-names>
            </name>
            <name name-style="western">
              <surname>Jain</surname>
              <given-names>P</given-names>
            </name>
            <name name-style="western">
              <surname>Chagnon</surname>
              <given-names>C</given-names>
            </name>
            <name name-style="western">
              <surname>Djamasbi</surname>
              <given-names>S</given-names>
            </name>
          </person-group>
          <article-title>Designing the empathetic research IoT network (ERIN) chatbot for mental health resources</article-title>
          <source>Proceedings of the HCI in Business, Government and Organizations: 8th International Conference, HCIBGO 2021, Held as Part of the 23rd HCI International Conference, HCII 2021</source>
          <year>2021</year>
          <conf-name>HCI 2021</conf-name>
          <conf-date>July 24-29, 2021</conf-date>
          <conf-loc>Virtual Event</conf-loc>
          <pub-id pub-id-type="doi">10.1007/978-3-030-77750-0_41</pub-id>
        </nlm-citation>
      </ref>
      <ref id="ref7">
        <label>7</label>
        <nlm-citation citation-type="journal">
          <person-group person-group-type="author">
            <name name-style="western">
              <surname>Li</surname>
              <given-names>L</given-names>
            </name>
            <name name-style="western">
              <surname>Peng</surname>
              <given-names>W</given-names>
            </name>
            <name name-style="western">
              <surname>Rheu</surname>
              <given-names>MM</given-names>
            </name>
          </person-group>
          <article-title>Factors predicting intentions of adoption and continued use of artificial intelligence chatbots for mental health: examining the role of UTAUT model, stigma, privacy concerns, and artificial intelligence hesitancy</article-title>
          <source>Telemed J E Health</source>
          <year>2024</year>
          <month>03</month>
          <day>01</day>
          <volume>30</volume>
          <issue>3</issue>
          <fpage>722</fpage>
          <lpage>30</lpage>
          <pub-id pub-id-type="doi">10.1089/tmj.2023.0313</pub-id>
          <pub-id pub-id-type="medline">37756224</pub-id>
        </nlm-citation>
      </ref>
      <ref id="ref8">
        <label>8</label>
        <nlm-citation citation-type="book">
          <person-group person-group-type="author">
            <name name-style="western">
              <surname>Hojat</surname>
              <given-names>M</given-names>
            </name>
          </person-group>
          <person-group person-group-type="editor">
            <name name-style="western">
              <surname>Hojat</surname>
              <given-names>M</given-names>
            </name>
          </person-group>
          <article-title>A definition and key features of empathy in patient care</article-title>
          <source>Empathy in Health Professions Education and Patient Care</source>
          <year>2016</year>
          <publisher-loc>Cham, Switzerland</publisher-loc>
          <publisher-name>Springer</publisher-name>
        </nlm-citation>
      </ref>
      <ref id="ref9">
        <label>9</label>
        <nlm-citation citation-type="journal">
          <person-group person-group-type="author">
            <name name-style="western">
              <surname>Koulouri</surname>
              <given-names>T</given-names>
            </name>
            <name name-style="western">
              <surname>Macredie</surname>
              <given-names>RD</given-names>
            </name>
            <name name-style="western">
              <surname>Olakitan</surname>
              <given-names>D</given-names>
            </name>
          </person-group>
          <article-title>Chatbots to support young adults’ mental health: an exploratory study of acceptability</article-title>
          <source>ACM Trans Interact Intell Syst</source>
          <year>2022</year>
          <month>07</month>
          <day>20</day>
          <volume>12</volume>
          <issue>2</issue>
          <fpage>1</fpage>
          <lpage>39</lpage>
          <pub-id pub-id-type="doi">10.1145/3485874</pub-id>
        </nlm-citation>
      </ref>
      <ref id="ref10">
        <label>10</label>
        <nlm-citation citation-type="journal">
          <person-group person-group-type="author">
            <name name-style="western">
              <surname>He</surname>
              <given-names>Y</given-names>
            </name>
            <name name-style="western">
              <surname>Yang</surname>
              <given-names>L</given-names>
            </name>
            <name name-style="western">
              <surname>Qian</surname>
              <given-names>C</given-names>
            </name>
            <name name-style="western">
              <surname>Li</surname>
              <given-names>T</given-names>
            </name>
            <name name-style="western">
              <surname>Su</surname>
              <given-names>Z</given-names>
            </name>
            <name name-style="western">
              <surname>Zhang</surname>
              <given-names>Q</given-names>
            </name>
            <name name-style="western">
              <surname>Hou</surname>
              <given-names>X</given-names>
            </name>
          </person-group>
          <article-title>Conversational agent interventions for mental health problems: systematic review and meta-analysis of randomized controlled trials</article-title>
          <source>J Med Internet Res</source>
          <year>2023</year>
          <month>04</month>
          <day>28</day>
          <volume>25</volume>
          <fpage>e43862</fpage>
          <comment>
            <ext-link ext-link-type="uri" xlink:type="simple" xlink:href="https://www.jmir.org/2023//e43862/"/>
          </comment>
          <pub-id pub-id-type="doi">10.2196/43862</pub-id>
          <pub-id pub-id-type="medline">37115595</pub-id>
          <pub-id pub-id-type="pii">v25i1e43862</pub-id>
          <pub-id pub-id-type="pmcid">PMC10182468</pub-id>
        </nlm-citation>
      </ref>
      <ref id="ref11">
        <label>11</label>
        <nlm-citation citation-type="confproc">
          <person-group person-group-type="author">
            <name name-style="western">
              <surname>Kraus</surname>
              <given-names>M</given-names>
            </name>
            <name name-style="western">
              <surname>Seldschopf</surname>
              <given-names>P</given-names>
            </name>
            <name name-style="western">
              <surname>Minker</surname>
              <given-names>W</given-names>
            </name>
          </person-group>
          <article-title>Towards the development of a trustworthy chatbot for mental health applications</article-title>
          <source>Proceedings of the 27th International Conference on MultiMedia Modeling</source>
          <year>2021</year>
          <conf-name>MMM 2021</conf-name>
          <conf-date>June 22-24, 2021</conf-date>
          <conf-loc>Prague, Czech Republic</conf-loc>
          <pub-id pub-id-type="doi">10.1007/978-3-030-67835-7_30</pub-id>
        </nlm-citation>
      </ref>
      <ref id="ref12">
        <label>12</label>
        <nlm-citation citation-type="confproc">
          <person-group person-group-type="author">
            <name name-style="western">
              <surname>Kallivalappil</surname>
              <given-names>N</given-names>
            </name>
            <name name-style="western">
              <surname>D’souza</surname>
              <given-names>K</given-names>
            </name>
            <name name-style="western">
              <surname>Deshmukh</surname>
              <given-names>A</given-names>
            </name>
            <name name-style="western">
              <surname>Kadam</surname>
              <given-names>C</given-names>
            </name>
            <name name-style="western">
              <surname>Sharma</surname>
              <given-names>N</given-names>
            </name>
          </person-group>
          <article-title>Empath.ai: a context-aware chatbot for emotional detection and support</article-title>
          <source>Proceedings of the 14th International Conference on Computing Communication and Networking Technologies</source>
          <year>2023</year>
          <conf-name>ICCCNT 2023</conf-name>
          <conf-date>July 6-8, 2023</conf-date>
          <conf-loc>Delhi, India</conf-loc>
          <pub-id pub-id-type="doi">10.1109/icccnt56998.2023.10306584</pub-id>
        </nlm-citation>
      </ref>
      <ref id="ref13">
        <label>13</label>
        <nlm-citation citation-type="confproc">
          <person-group person-group-type="author">
            <name name-style="western">
              <surname>Lin</surname>
              <given-names>S</given-names>
            </name>
            <name name-style="western">
              <surname>Lin</surname>
              <given-names>L</given-names>
            </name>
            <name name-style="western">
              <surname>Hou</surname>
              <given-names>C</given-names>
            </name>
            <name name-style="western">
              <surname>Chen</surname>
              <given-names>B</given-names>
            </name>
            <name name-style="western">
              <surname>Li</surname>
              <given-names>J</given-names>
            </name>
            <name name-style="western">
              <surname>Ni</surname>
              <given-names>S</given-names>
            </name>
          </person-group>
          <article-title>Empathy-based communication framework for chatbots: a mental health chatbot application and evaluation</article-title>
          <source>Proceedings of the 11th International Conference on Human-Agent Interaction</source>
          <year>2023</year>
          <conf-name>HAI '23</conf-name>
          <conf-date>December 4-7, 2023</conf-date>
          <conf-loc>Gothenburg, Sweden</conf-loc>
          <pub-id pub-id-type="doi">10.1145/3623809.3623865</pub-id>
        </nlm-citation>
      </ref>
      <ref id="ref14">
        <label>14</label>
        <nlm-citation citation-type="journal">
          <person-group person-group-type="author">
            <name name-style="western">
              <surname>Boucher</surname>
              <given-names>EM</given-names>
            </name>
            <name name-style="western">
              <surname>Harake</surname>
              <given-names>NR</given-names>
            </name>
            <name name-style="western">
              <surname>Ward</surname>
              <given-names>HE</given-names>
            </name>
            <name name-style="western">
              <surname>Stoeckl</surname>
              <given-names>SE</given-names>
            </name>
            <name name-style="western">
              <surname>Vargas</surname>
              <given-names>J</given-names>
            </name>
            <name name-style="western">
              <surname>Minkel</surname>
              <given-names>J</given-names>
            </name>
            <name name-style="western">
              <surname>Parks</surname>
              <given-names>AC</given-names>
            </name>
            <name name-style="western">
              <surname>Zilca</surname>
              <given-names>R</given-names>
            </name>
          </person-group>
          <article-title>Artificially intelligent chatbots in digital mental health interventions: a review</article-title>
          <source>Expert Rev Med Devices</source>
          <year>2021</year>
          <month>12</month>
          <volume>18</volume>
          <issue>sup1</issue>
          <fpage>37</fpage>
          <lpage>49</lpage>
          <comment>
            <ext-link ext-link-type="uri" xlink:type="simple" xlink:href="http://paperpile.com/b/lFnvVd/oyDj"/>
          </comment>
          <pub-id pub-id-type="doi">10.1080/17434440.2021.2013200</pub-id>
          <pub-id pub-id-type="medline">34872429</pub-id>
        </nlm-citation>
      </ref>
      <ref id="ref15">
        <label>15</label>
        <nlm-citation citation-type="journal">
          <person-group person-group-type="author">
            <name name-style="western">
              <surname>Sweeney</surname>
              <given-names>C</given-names>
            </name>
            <name name-style="western">
              <surname>Potts</surname>
              <given-names>C</given-names>
            </name>
            <name name-style="western">
              <surname>Ennis</surname>
              <given-names>E</given-names>
            </name>
            <name name-style="western">
              <surname>Bond</surname>
              <given-names>R</given-names>
            </name>
            <name name-style="western">
              <surname>Mulvenna</surname>
              <given-names>MD</given-names>
            </name>
            <name name-style="western">
              <surname>O’neill</surname>
              <given-names>S</given-names>
            </name>
            <name name-style="western">
              <surname>Malcolm</surname>
              <given-names>M</given-names>
            </name>
            <name name-style="western">
              <surname>Kuosmanen</surname>
              <given-names>L</given-names>
            </name>
            <name name-style="western">
              <surname>Kostenius</surname>
              <given-names>C</given-names>
            </name>
            <name name-style="western">
              <surname>Vakaloudis</surname>
              <given-names>A</given-names>
            </name>
            <name name-style="western">
              <surname>Mcconvey</surname>
              <given-names>G</given-names>
            </name>
            <name name-style="western">
              <surname>Turkington</surname>
              <given-names>R</given-names>
            </name>
            <name name-style="western">
              <surname>Hanna</surname>
              <given-names>D</given-names>
            </name>
            <name name-style="western">
              <surname>Nieminen</surname>
              <given-names>H</given-names>
            </name>
            <name name-style="western">
              <surname>Vartiainen</surname>
              <given-names>AK</given-names>
            </name>
            <name name-style="western">
              <surname>Robertson</surname>
              <given-names>A</given-names>
            </name>
            <name name-style="western">
              <surname>Mctear</surname>
              <given-names>MF</given-names>
            </name>
          </person-group>
          <article-title>Can chatbots help support a person’s mental health? Perceptions and views from mental healthcare professionals and experts</article-title>
          <source>ACM Trans Comput Healthcare</source>
          <year>2021</year>
          <month>07</month>
          <day>15</day>
          <volume>2</volume>
          <issue>3</issue>
          <fpage>1</fpage>
          <lpage>15</lpage>
          <pub-id pub-id-type="doi">10.1145/3453175</pub-id>
        </nlm-citation>
      </ref>
      <ref id="ref16">
        <label>16</label>
        <nlm-citation citation-type="journal">
          <person-group person-group-type="author">
            <name name-style="western">
              <surname>Daley</surname>
              <given-names>K</given-names>
            </name>
            <name name-style="western">
              <surname>Hungerbuehler</surname>
              <given-names>I</given-names>
            </name>
            <name name-style="western">
              <surname>Cavanagh</surname>
              <given-names>K</given-names>
            </name>
            <name name-style="western">
              <surname>Claro</surname>
              <given-names>HG</given-names>
            </name>
            <name name-style="western">
              <surname>Swinton</surname>
              <given-names>PA</given-names>
            </name>
            <name name-style="western">
              <surname>Kapps</surname>
              <given-names>M</given-names>
            </name>
          </person-group>
          <article-title>Preliminary evaluation of the engagement and effectiveness of a mental health chatbot</article-title>
          <source>Front Digit Health</source>
          <year>2020</year>
          <month>11</month>
          <day>30</day>
          <volume>2</volume>
          <fpage>576361</fpage>
          <comment>
            <ext-link ext-link-type="uri" xlink:type="simple" xlink:href="https://europepmc.org/abstract/MED/34713049"/>
          </comment>
          <pub-id pub-id-type="doi">10.3389/fdgth.2020.576361</pub-id>
          <pub-id pub-id-type="medline">34713049</pub-id>
          <pub-id pub-id-type="pmcid">PMC8521874</pub-id>
        </nlm-citation>
      </ref>
      <ref id="ref17">
        <label>17</label>
        <nlm-citation citation-type="journal">
          <person-group person-group-type="author">
            <name name-style="western">
              <surname>Li</surname>
              <given-names>H</given-names>
            </name>
            <name name-style="western">
              <surname>Zhang</surname>
              <given-names>R</given-names>
            </name>
            <name name-style="western">
              <surname>Lee</surname>
              <given-names>YC</given-names>
            </name>
            <name name-style="western">
              <surname>Kraut</surname>
              <given-names>RE</given-names>
            </name>
            <name name-style="western">
              <surname>Mohr</surname>
              <given-names>DC</given-names>
            </name>
          </person-group>
          <article-title>Systematic review and meta-analysis of AI-based conversational agents for promoting mental health and well-being</article-title>
          <source>NPJ Digit Med</source>
          <year>2023</year>
          <month>12</month>
          <day>19</day>
          <volume>6</volume>
          <issue>1</issue>
          <fpage>236</fpage>
          <comment>
            <ext-link ext-link-type="uri" xlink:type="simple" xlink:href="https://doi.org/10.1038/s41746-023-00979-5"/>
          </comment>
          <pub-id pub-id-type="doi">10.1038/s41746-023-00979-5</pub-id>
          <pub-id pub-id-type="medline">38114588</pub-id>
          <pub-id pub-id-type="pii">10.1038/s41746-023-00979-5</pub-id>
          <pub-id pub-id-type="pmcid">PMC10730549</pub-id>
        </nlm-citation>
      </ref>
      <ref id="ref18">
        <label>18</label>
        <nlm-citation citation-type="journal">
          <person-group person-group-type="author">
            <name name-style="western">
              <surname>Gaffney</surname>
              <given-names>H</given-names>
            </name>
            <name name-style="western">
              <surname>Mansell</surname>
              <given-names>W</given-names>
            </name>
            <name name-style="western">
              <surname>Tai</surname>
              <given-names>S</given-names>
            </name>
          </person-group>
          <article-title>Conversational agents in the treatment of mental health problems: mixed-method systematic review</article-title>
          <source>JMIR Ment Health</source>
          <year>2019</year>
          <month>10</month>
          <day>18</day>
          <volume>6</volume>
          <issue>10</issue>
          <fpage>e14166</fpage>
          <comment>
            <ext-link ext-link-type="uri" xlink:type="simple" xlink:href="https://mental.jmir.org/2019/10/e14166/"/>
          </comment>
          <pub-id pub-id-type="doi">10.2196/14166</pub-id>
          <pub-id pub-id-type="medline">31628789</pub-id>
          <pub-id pub-id-type="pii">v6i10e14166</pub-id>
          <pub-id pub-id-type="pmcid">PMC6914342</pub-id>
        </nlm-citation>
      </ref>
      <ref id="ref19">
        <label>19</label>
        <nlm-citation citation-type="journal">
          <person-group person-group-type="author">
            <name name-style="western">
              <surname>Abd-Alrazaq</surname>
              <given-names>AA</given-names>
            </name>
            <name name-style="western">
              <surname>Alajlani</surname>
              <given-names>M</given-names>
            </name>
            <name name-style="western">
              <surname>Ali</surname>
              <given-names>N</given-names>
            </name>
            <name name-style="western">
              <surname>Denecke</surname>
              <given-names>K</given-names>
            </name>
            <name name-style="western">
              <surname>Bewick</surname>
              <given-names>BM</given-names>
            </name>
            <name name-style="western">
              <surname>Househ</surname>
              <given-names>M</given-names>
            </name>
          </person-group>
          <article-title>Perceptions and opinions of patients about mental health chatbots: scoping review</article-title>
          <source>J Med Internet Res</source>
          <year>2021</year>
          <month>01</month>
          <day>13</day>
          <volume>23</volume>
          <issue>1</issue>
          <fpage>e17828</fpage>
          <comment>
            <ext-link ext-link-type="uri" xlink:type="simple" xlink:href="https://www.jmir.org/2021/1/e17828/"/>
          </comment>
          <pub-id pub-id-type="doi">10.2196/17828</pub-id>
          <pub-id pub-id-type="medline">33439133</pub-id>
          <pub-id pub-id-type="pii">v23i1e17828</pub-id>
          <pub-id pub-id-type="pmcid">PMC7840290</pub-id>
        </nlm-citation>
      </ref>
      <ref id="ref20">
        <label>20</label>
        <nlm-citation citation-type="web">
          <person-group person-group-type="author">
            <name name-style="western">
              <surname>Behrens</surname>
              <given-names>S</given-names>
            </name>
          </person-group>
          <article-title>The history and evolution of conversational AI</article-title>
          <source>Fabric</source>
          <year>2021</year>
          <month>04</month>
          <day>17</day>
          <access-date>2023-03-13</access-date>
          <comment>
            <ext-link ext-link-type="uri" xlink:type="simple" xlink:href="https://gyant.com/the-history-and-evolution-of-conversational-ai/">https://gyant.com/the-history-and-evolution-of-conversational-ai/</ext-link>
          </comment>
        </nlm-citation>
      </ref>
      <ref id="ref21">
        <label>21</label>
        <nlm-citation citation-type="journal">
          <person-group person-group-type="author">
            <name name-style="western">
              <surname>Jiang</surname>
              <given-names>Q</given-names>
            </name>
            <name name-style="western">
              <surname>Zhang</surname>
              <given-names>Y</given-names>
            </name>
            <name name-style="western">
              <surname>Pian</surname>
              <given-names>W</given-names>
            </name>
          </person-group>
          <article-title>Chatbot as an emergency exist: mediated empathy for resilience via human-AI interaction during the COVID-19 pandemic</article-title>
          <source>Inf Process Manag</source>
          <year>2022</year>
          <month>11</month>
          <volume>59</volume>
          <issue>6</issue>
          <fpage>103074</fpage>
          <comment>
            <ext-link ext-link-type="uri" xlink:type="simple" xlink:href="https://europepmc.org/abstract/MED/36059428"/>
          </comment>
          <pub-id pub-id-type="doi">10.1016/j.ipm.2022.103074</pub-id>
          <pub-id pub-id-type="medline">36059428</pub-id>
          <pub-id pub-id-type="pii">S0306-4573(22)00175-3</pub-id>
          <pub-id pub-id-type="pmcid">PMC9428597</pub-id>
        </nlm-citation>
      </ref>
      <ref id="ref22">
        <label>22</label>
        <nlm-citation citation-type="confproc">
          <person-group person-group-type="author">
            <name name-style="western">
              <surname>Brocki</surname>
              <given-names>L</given-names>
            </name>
            <name name-style="western">
              <surname>Dyer</surname>
              <given-names>GC</given-names>
            </name>
            <name name-style="western">
              <surname>Gładka</surname>
              <given-names>A</given-names>
            </name>
            <name name-style="western">
              <surname>Chung</surname>
              <given-names>NC</given-names>
            </name>
          </person-group>
          <article-title>Deep learning mental health dialogue system</article-title>
          <source>Proceedings of the IEEE International Conference on Big Data and Smart Computing</source>
          <year>2023</year>
          <conf-name>BigComp 2023</conf-name>
          <conf-date>February 13-16, 2023</conf-date>
          <conf-loc>Jeju, Republic of Korea</conf-loc>
          <pub-id pub-id-type="doi">10.1109/bigcomp57234.2023.00097</pub-id>
        </nlm-citation>
      </ref>
      <ref id="ref23">
        <label>23</label>
        <nlm-citation citation-type="journal">
          <person-group person-group-type="author">
            <name name-style="western">
              <surname>Trappey</surname>
              <given-names>AJ</given-names>
            </name>
            <name name-style="western">
              <surname>Lin</surname>
              <given-names>AP</given-names>
            </name>
            <name name-style="western">
              <surname>Hsu</surname>
              <given-names>KY</given-names>
            </name>
            <name name-style="western">
              <surname>Trappey</surname>
              <given-names>CV</given-names>
            </name>
            <name name-style="western">
              <surname>Tu</surname>
              <given-names>KL</given-names>
            </name>
          </person-group>
          <article-title>Development of an empathy-centric counseling chatbot system capable of sentimental dialogue analysis</article-title>
          <source>Processes</source>
          <year>2022</year>
          <month>05</month>
          <day>08</day>
          <volume>10</volume>
          <issue>5</issue>
          <fpage>930</fpage>
          <pub-id pub-id-type="doi">10.3390/pr10050930</pub-id>
        </nlm-citation>
      </ref>
      <ref id="ref24">
        <label>24</label>
        <nlm-citation citation-type="confproc">
          <person-group person-group-type="author">
            <name name-style="western">
              <surname>Ghandeharioun</surname>
              <given-names>A</given-names>
            </name>
            <name name-style="western">
              <surname>McDuff</surname>
              <given-names>D</given-names>
            </name>
            <name name-style="western">
              <surname>Czerwinski</surname>
              <given-names>M</given-names>
            </name>
            <name name-style="western">
              <surname>Rowan</surname>
              <given-names>K</given-names>
            </name>
          </person-group>
          <article-title>EMMA: an emotion-aware wellbeing chatbot</article-title>
          <source>Proceedings of the 8th International Conference on Affective Computing and Intelligent Interaction</source>
          <year>2019</year>
          <conf-name>ACII 2019</conf-name>
          <conf-date>September 3-6, 2019</conf-date>
          <conf-loc>Cambridge, UK</conf-loc>
          <pub-id pub-id-type="doi">10.1109/acii.2019.8925455</pub-id>
        </nlm-citation>
      </ref>
      <ref id="ref25">
        <label>25</label>
        <nlm-citation citation-type="journal">
          <person-group person-group-type="author">
            <name name-style="western">
              <surname>Meng</surname>
              <given-names>J</given-names>
            </name>
            <name name-style="western">
              <surname>Dai</surname>
              <given-names>Y</given-names>
            </name>
          </person-group>
          <article-title>Emotional support from AI chatbots: should a supportive partner self-disclose or not?</article-title>
          <source>J Comput Mediat Commun</source>
          <year>2021</year>
          <month>05</month>
          <day>19</day>
          <volume>26</volume>
          <issue>4</issue>
          <fpage>207</fpage>
          <lpage>22</lpage>
          <pub-id pub-id-type="doi">10.1093/jcmc/zmab005</pub-id>
        </nlm-citation>
      </ref>
      <ref id="ref26">
        <label>26</label>
        <nlm-citation citation-type="confproc">
          <person-group person-group-type="author">
            <name name-style="western">
              <surname>Goel</surname>
              <given-names>R</given-names>
            </name>
            <name name-style="western">
              <surname>Vashisht</surname>
              <given-names>S</given-names>
            </name>
            <name name-style="western">
              <surname>Dhanda</surname>
              <given-names>A</given-names>
            </name>
            <name name-style="western">
              <surname>Susan</surname>
              <given-names>S</given-names>
            </name>
          </person-group>
          <article-title>An empathetic conversational agent with attentional mechanism</article-title>
          <source>Proceedings of the International Conference on Computer Communication and Informatics</source>
          <year>2021</year>
          <conf-name>ICCCI 2021</conf-name>
          <conf-date>January 27-29, 2021</conf-date>
          <conf-loc>Coimbatore, India</conf-loc>
          <pub-id pub-id-type="doi">10.1109/iccci50826.2021.9402337</pub-id>
        </nlm-citation>
      </ref>
      <ref id="ref27">
        <label>27</label>
        <nlm-citation citation-type="journal">
          <person-group person-group-type="author">
            <name name-style="western">
              <surname>Adikari</surname>
              <given-names>A</given-names>
            </name>
            <name name-style="western">
              <surname>de Silva</surname>
              <given-names>D</given-names>
            </name>
            <name name-style="western">
              <surname>Moraliyage</surname>
              <given-names>H</given-names>
            </name>
            <name name-style="western">
              <surname>Alahakoon</surname>
              <given-names>D</given-names>
            </name>
            <name name-style="western">
              <surname>Wong</surname>
              <given-names>J</given-names>
            </name>
            <name name-style="western">
              <surname>Gancarz</surname>
              <given-names>M</given-names>
            </name>
            <name name-style="western">
              <surname>Chackochan</surname>
              <given-names>S</given-names>
            </name>
            <name name-style="western">
              <surname>Park</surname>
              <given-names>B</given-names>
            </name>
            <name name-style="western">
              <surname>Heo</surname>
              <given-names>R</given-names>
            </name>
            <name name-style="western">
              <surname>Leung</surname>
              <given-names>Y</given-names>
            </name>
          </person-group>
          <article-title>Empathic conversational agents for real-time monitoring and co-facilitation of patient-centered healthcare</article-title>
          <source>Future Gener Comput Syst</source>
          <year>2022</year>
          <month>01</month>
          <volume>126</volume>
          <fpage>318</fpage>
          <lpage>29</lpage>
          <pub-id pub-id-type="doi">10.1016/j.future.2021.08.015</pub-id>
        </nlm-citation>
      </ref>
      <ref id="ref28">
        <label>28</label>
        <nlm-citation citation-type="confproc">
          <person-group person-group-type="author">
            <name name-style="western">
              <surname>Beredo</surname>
              <given-names>JL</given-names>
            </name>
            <name name-style="western">
              <surname>Ong</surname>
              <given-names>EC</given-names>
            </name>
          </person-group>
          <article-title>A hybrid response generation model for an empathetic conversational agent</article-title>
          <source>Proceedings of the International Conference on Asian Language Processing</source>
          <year>2022</year>
          <conf-name>IALP 2022</conf-name>
          <conf-date>October 27-28, 2022</conf-date>
          <conf-loc>Singapore, Singapore</conf-loc>
          <pub-id pub-id-type="doi">10.1109/ialp57159.2022.9961311</pub-id>
        </nlm-citation>
      </ref>
      <ref id="ref29">
        <label>29</label>
        <nlm-citation citation-type="journal">
          <person-group person-group-type="author">
            <name name-style="western">
              <surname>Rathnayaka</surname>
              <given-names>P</given-names>
            </name>
            <name name-style="western">
              <surname>Mills</surname>
              <given-names>N</given-names>
            </name>
            <name name-style="western">
              <surname>Burnett</surname>
              <given-names>D</given-names>
            </name>
            <name name-style="western">
              <surname>De Silva</surname>
              <given-names>D</given-names>
            </name>
            <name name-style="western">
              <surname>Alahakoon</surname>
              <given-names>D</given-names>
            </name>
            <name name-style="western">
              <surname>Gray</surname>
              <given-names>R</given-names>
            </name>
          </person-group>
          <article-title>A mental health chatbot with cognitive skills for personalised behavioural activation and remote health monitoring</article-title>
          <source>Sensors (Basel)</source>
          <year>2022</year>
          <month>05</month>
          <day>11</day>
          <volume>22</volume>
          <issue>10</issue>
          <fpage>3653</fpage>
          <comment>
            <ext-link ext-link-type="uri" xlink:type="simple" xlink:href="https://www.mdpi.com/resolver?pii=s22103653"/>
          </comment>
          <pub-id pub-id-type="doi">10.3390/s22103653</pub-id>
          <pub-id pub-id-type="medline">35632061</pub-id>
          <pub-id pub-id-type="pii">s22103653</pub-id>
          <pub-id pub-id-type="pmcid">PMC9148050</pub-id>
        </nlm-citation>
      </ref>
      <ref id="ref30">
        <label>30</label>
        <nlm-citation citation-type="journal">
          <person-group person-group-type="author">
            <name name-style="western">
              <surname>Morris</surname>
              <given-names>RR</given-names>
            </name>
            <name name-style="western">
              <surname>Kouddous</surname>
              <given-names>K</given-names>
            </name>
            <name name-style="western">
              <surname>Kshirsagar</surname>
              <given-names>R</given-names>
            </name>
            <name name-style="western">
              <surname>Schueller</surname>
              <given-names>SM</given-names>
            </name>
          </person-group>
          <article-title>Towards an artificially empathic conversational agent for mental health applications: system design and user perceptions</article-title>
          <source>J Med Internet Res</source>
          <year>2018</year>
          <month>06</month>
          <day>26</day>
          <volume>20</volume>
          <issue>6</issue>
          <fpage>e10148</fpage>
          <comment>
            <ext-link ext-link-type="uri" xlink:type="simple" xlink:href="https://www.jmir.org/2018/6/e10148/"/>
          </comment>
          <pub-id pub-id-type="doi">10.2196/10148</pub-id>
          <pub-id pub-id-type="medline">29945856</pub-id>
          <pub-id pub-id-type="pii">v20i6e10148</pub-id>
          <pub-id pub-id-type="pmcid">PMC6039770</pub-id>
        </nlm-citation>
      </ref>
      <ref id="ref31">
        <label>31</label>
        <nlm-citation citation-type="confproc">
          <person-group person-group-type="author">
            <name name-style="western">
              <surname>Ghandeharioun</surname>
              <given-names>A</given-names>
            </name>
            <name name-style="western">
              <surname>McDuff</surname>
              <given-names>D</given-names>
            </name>
            <name name-style="western">
              <surname>Czerwinski</surname>
              <given-names>M</given-names>
            </name>
            <name name-style="western">
              <surname>Rowan</surname>
              <given-names>K</given-names>
            </name>
          </person-group>
          <article-title>Towards understanding emotional intelligence for behavior change chatbots</article-title>
          <source>Proceedings of the 8th International Conference on Affective Computing and Intelligent Interaction</source>
          <year>2019</year>
          <conf-name>ACII 2019</conf-name>
          <conf-date>September 3-6, 2019</conf-date>
          <conf-loc>Cambridge, UK</conf-loc>
          <pub-id pub-id-type="doi">10.1109/acii.2019.8925433</pub-id>
        </nlm-citation>
      </ref>
      <ref id="ref32">
        <label>32</label>
        <nlm-citation citation-type="confproc">
          <person-group person-group-type="author">
            <name name-style="western">
              <surname>Saha</surname>
              <given-names>T</given-names>
            </name>
            <name name-style="western">
              <surname>Gakhreja</surname>
              <given-names>V</given-names>
            </name>
            <name name-style="western">
              <surname>Das</surname>
              <given-names>AS</given-names>
            </name>
            <name name-style="western">
              <surname>Chakraborty</surname>
              <given-names>S</given-names>
            </name>
            <name name-style="western">
              <surname>Saha</surname>
              <given-names>S</given-names>
            </name>
          </person-group>
          <article-title>Towards motivational and empathetic response generation in online mental health support</article-title>
          <source>Proceedings of the 45th International ACM SIGIR Conference on Research and Development in Information Retrieval</source>
          <year>2022</year>
          <conf-name>SIGIR '22</conf-name>
          <conf-date>July 11-15, 2022</conf-date>
          <conf-loc>Madrid, Spain</conf-loc>
          <pub-id pub-id-type="doi">10.1145/3477495.3531912</pub-id>
        </nlm-citation>
      </ref>
      <ref id="ref33">
        <label>33</label>
        <nlm-citation citation-type="confproc">
          <person-group person-group-type="author">
            <name name-style="western">
              <surname>Agnihotri</surname>
              <given-names>M</given-names>
            </name>
            <name name-style="western">
              <surname>Rao</surname>
              <given-names>SB</given-names>
            </name>
            <name name-style="western">
              <surname>Jayagopi</surname>
              <given-names>DB</given-names>
            </name>
            <name name-style="western">
              <surname>Hebbar</surname>
              <given-names>S</given-names>
            </name>
            <name name-style="western">
              <surname>Rasipuram</surname>
              <given-names>S</given-names>
            </name>
            <name name-style="western">
              <surname>Maitra</surname>
              <given-names>A</given-names>
            </name>
            <name name-style="western">
              <surname>Sengupta</surname>
              <given-names>S</given-names>
            </name>
          </person-group>
          <article-title>Towards generating topic-driven and affective responses to assist mental wellness</article-title>
          <source>Proceedings of the 25th International Conference on Pattern Recognition</source>
          <year>2021</year>
          <conf-name>ICPR 2021</conf-name>
          <conf-date>January 10-15, 2021</conf-date>
          <conf-loc>Milan, Italy</conf-loc>
          <pub-id pub-id-type="doi">10.1007/978-3-030-68790-8_11</pub-id>
        </nlm-citation>
      </ref>
      <ref id="ref34">
        <label>34</label>
        <nlm-citation citation-type="confproc">
          <person-group person-group-type="author">
            <name name-style="western">
              <surname>Alazraki</surname>
              <given-names>L</given-names>
            </name>
            <name name-style="western">
              <surname>Ghachem</surname>
              <given-names>A</given-names>
            </name>
            <name name-style="western">
              <surname>Polydorou</surname>
              <given-names>N</given-names>
            </name>
            <name name-style="western">
              <surname>Khosmood</surname>
              <given-names>F</given-names>
            </name>
            <name name-style="western">
              <surname>Edalat</surname>
              <given-names>A</given-names>
            </name>
          </person-group>
          <article-title>An empathetic AI coach for self-attachment therapy</article-title>
          <source>Proceedings of the IEEE Third International Conference on Cognitive Machine Intelligence</source>
          <year>2021</year>
          <conf-name>CogMI 2021</conf-name>
          <conf-date>December 13-15, 2021</conf-date>
          <conf-loc>Atlanta, GA</conf-loc>
          <pub-id pub-id-type="doi">10.1109/cogmi52975.2021.00019</pub-id>
        </nlm-citation>
      </ref>
      <ref id="ref35">
        <label>35</label>
        <nlm-citation citation-type="confproc">
          <person-group person-group-type="author">
            <name name-style="western">
              <surname>Gundavarapu</surname>
              <given-names>MR</given-names>
            </name>
            <name name-style="western">
              <surname>Koundinya</surname>
              <given-names>GS</given-names>
            </name>
            <name name-style="western">
              <surname>Sai</surname>
              <given-names>TB</given-names>
            </name>
            <name name-style="western">
              <surname>Sree</surname>
              <given-names>GK</given-names>
            </name>
          </person-group>
          <article-title>Empathic chatbot: emotional astuteness for mental health well-being</article-title>
          <source>Proceedings of the 7th International Conference on Computing in Engineering &#38; Technology</source>
          <year>2022</year>
          <conf-name>ICCET 2022</conf-name>
          <conf-date>February 25-27, 2022</conf-date>
          <conf-loc>Virtual Event</conf-loc>
          <pub-id pub-id-type="doi">10.1007/978-981-19-2719-5_65</pub-id>
        </nlm-citation>
      </ref>
      <ref id="ref36">
        <label>36</label>
        <nlm-citation citation-type="confproc">
          <person-group person-group-type="author">
            <name name-style="western">
              <surname>Mishra</surname>
              <given-names>K</given-names>
            </name>
            <name name-style="western">
              <surname>Priya</surname>
              <given-names>P</given-names>
            </name>
            <name name-style="western">
              <surname>Ekbal</surname>
              <given-names>A</given-names>
            </name>
          </person-group>
          <article-title>Help me heal: a reinforced polite and empathetic mental health and legal counseling dialogue system for crime victims</article-title>
          <source>Proceedings of the 37th AAAI Conference on Artificial Intelligence</source>
          <year>2023</year>
          <conf-name>AAAI 2023</conf-name>
          <conf-date>February 7-14, 2023</conf-date>
          <conf-loc>Washington, DC</conf-loc>
          <pub-id pub-id-type="doi">10.1609/aaai.v37i12.26685</pub-id>
        </nlm-citation>
      </ref>
      <ref id="ref37">
        <label>37</label>
        <nlm-citation citation-type="journal">
          <person-group person-group-type="author">
            <name name-style="western">
              <surname>Gotschall</surname>
              <given-names>T</given-names>
            </name>
          </person-group>
          <article-title>EndNote 20 desktop version</article-title>
          <source>J Med Libr Assoc</source>
          <year>2021</year>
          <month>07</month>
          <day>01</day>
          <volume>109</volume>
          <issue>3</issue>
          <fpage>520</fpage>
          <lpage>2</lpage>
          <comment>
            <ext-link ext-link-type="uri" xlink:type="simple" xlink:href="https://europepmc.org/abstract/MED/34629985"/>
          </comment>
          <pub-id pub-id-type="doi">10.5195/jmla.2021.1260</pub-id>
          <pub-id pub-id-type="medline">34629985</pub-id>
          <pub-id pub-id-type="pii">jmla.2021.1260</pub-id>
          <pub-id pub-id-type="pmcid">PMC8485940</pub-id>
        </nlm-citation>
      </ref>
      <ref id="ref38">
        <label>38</label>
        <nlm-citation citation-type="book">
          <person-group person-group-type="author">
            <name name-style="western">
              <surname>Moola</surname>
              <given-names>S</given-names>
            </name>
            <name name-style="western">
              <surname>Munn</surname>
              <given-names>Z</given-names>
            </name>
            <name name-style="western">
              <surname>Tufanaru</surname>
              <given-names>C</given-names>
            </name>
            <name name-style="western">
              <surname>Aromataris</surname>
              <given-names>E</given-names>
            </name>
            <name name-style="western">
              <surname>Sears</surname>
              <given-names>K</given-names>
            </name>
            <name name-style="western">
              <surname>Sfetcu</surname>
              <given-names>R</given-names>
            </name>
            <name name-style="western">
              <surname>Currie</surname>
              <given-names>M</given-names>
            </name>
            <name name-style="western">
              <surname>Lisy</surname>
              <given-names>K</given-names>
            </name>
            <name name-style="western">
              <surname>Qureshi</surname>
              <given-names>R</given-names>
            </name>
            <name name-style="western">
              <surname>Mattis</surname>
              <given-names>P</given-names>
            </name>
            <name name-style="western">
              <surname>Mu</surname>
              <given-names>PF</given-names>
            </name>
          </person-group>
          <person-group person-group-type="editor">
            <name name-style="western">
              <surname>Aromataris</surname>
              <given-names>E</given-names>
            </name>
            <name name-style="western">
              <surname>Lockwood</surname>
              <given-names>C</given-names>
            </name>
            <name name-style="western">
              <surname>Porritt</surname>
              <given-names>K</given-names>
            </name>
            <name name-style="western">
              <surname>Pilla</surname>
              <given-names>B</given-names>
            </name>
            <name name-style="western">
              <surname>Jordan</surname>
              <given-names>Z</given-names>
            </name>
          </person-group>
          <article-title>Systematic reviews of etiology and risk</article-title>
          <source>JBI Manual for Evidence Synthesis</source>
          <year>2024</year>
          <publisher-loc>Adelaide, Australia</publisher-loc>
          <publisher-name>Joanna Briggs Institute</publisher-name>
        </nlm-citation>
      </ref>
      <ref id="ref39">
        <label>39</label>
        <nlm-citation citation-type="journal">
          <person-group person-group-type="author">
            <name name-style="western">
              <surname>Rogers</surname>
              <given-names>CR</given-names>
            </name>
          </person-group>
          <article-title>Empathic: an unappreciated way of being</article-title>
          <source>Couns Psychol</source>
          <year>2016</year>
          <month>09</month>
          <day>04</day>
          <volume>5</volume>
          <issue>2</issue>
          <fpage>2</fpage>
          <lpage>10</lpage>
          <pub-id pub-id-type="doi">10.1177/001100007500500202</pub-id>
        </nlm-citation>
      </ref>
      <ref id="ref40">
        <label>40</label>
        <nlm-citation citation-type="journal">
          <person-group person-group-type="author">
            <name name-style="western">
              <surname>Elliott</surname>
              <given-names>R</given-names>
            </name>
            <name name-style="western">
              <surname>Bohart</surname>
              <given-names>AC</given-names>
            </name>
            <name name-style="western">
              <surname>Watson</surname>
              <given-names>JC</given-names>
            </name>
            <name name-style="western">
              <surname>Murphy</surname>
              <given-names>D</given-names>
            </name>
          </person-group>
          <article-title>Therapist empathy and client outcome: an updated meta-analysis</article-title>
          <source>Psychotherapy (Chic)</source>
          <year>2018</year>
          <month>12</month>
          <volume>55</volume>
          <issue>4</issue>
          <fpage>399</fpage>
          <lpage>410</lpage>
          <comment>
            <ext-link ext-link-type="uri" xlink:type="simple" xlink:href="https://strathprints.strath.ac.uk/66200/"/>
          </comment>
          <pub-id pub-id-type="doi">10.1037/pst0000175</pub-id>
          <pub-id pub-id-type="medline">30335453</pub-id>
          <pub-id pub-id-type="pii">2018-51673-006</pub-id>
        </nlm-citation>
      </ref>
      <ref id="ref41">
        <label>41</label>
        <nlm-citation citation-type="journal">
          <person-group person-group-type="author">
            <name name-style="western">
              <surname>Barrett-Lennard</surname>
              <given-names>GT</given-names>
            </name>
          </person-group>
          <article-title>The empathy cycle: refinement of a nuclear concept</article-title>
          <source>J Counsel Psychol</source>
          <year>1981</year>
          <volume>28</volume>
          <issue>2</issue>
          <fpage>91</fpage>
          <lpage>100</lpage>
          <pub-id pub-id-type="doi">10.1037//0022-0167.28.2.91</pub-id>
        </nlm-citation>
      </ref>
      <ref id="ref42">
        <label>42</label>
        <nlm-citation citation-type="journal">
          <person-group person-group-type="author">
            <name name-style="western">
              <surname>Vaidyam</surname>
              <given-names>AN</given-names>
            </name>
            <name name-style="western">
              <surname>Wisniewski</surname>
              <given-names>H</given-names>
            </name>
            <name name-style="western">
              <surname>Halamka</surname>
              <given-names>JD</given-names>
            </name>
            <name name-style="western">
              <surname>Kashavan</surname>
              <given-names>MS</given-names>
            </name>
            <name name-style="western">
              <surname>Torous</surname>
              <given-names>JB</given-names>
            </name>
          </person-group>
          <article-title>Chatbots and conversational agents in mental health: a review of the psychiatric landscape</article-title>
          <source>Can J Psychiatry</source>
          <year>2019</year>
          <month>07</month>
          <volume>64</volume>
          <issue>7</issue>
          <fpage>456</fpage>
          <lpage>64</lpage>
          <comment>
            <ext-link ext-link-type="uri" xlink:type="simple" xlink:href="https://europepmc.org/abstract/MED/30897957"/>
          </comment>
          <pub-id pub-id-type="doi">10.1177/0706743719828977</pub-id>
          <pub-id pub-id-type="medline">30897957</pub-id>
          <pub-id pub-id-type="pmcid">PMC6610568</pub-id>
        </nlm-citation>
      </ref>
      <ref id="ref43">
        <label>43</label>
        <nlm-citation citation-type="journal">
          <person-group person-group-type="author">
            <name name-style="western">
              <surname>Xu</surname>
              <given-names>X</given-names>
            </name>
            <name name-style="western">
              <surname>Yao</surname>
              <given-names>B</given-names>
            </name>
            <name name-style="western">
              <surname>Dong</surname>
              <given-names>Y</given-names>
            </name>
            <name name-style="western">
              <surname>Gabriel</surname>
              <given-names>S</given-names>
            </name>
            <name name-style="western">
              <surname>Yu</surname>
              <given-names>H</given-names>
            </name>
            <name name-style="western">
              <surname>Hendler</surname>
              <given-names>J</given-names>
            </name>
            <name name-style="western">
              <surname>Ghassemi</surname>
              <given-names>M</given-names>
            </name>
            <name name-style="western">
              <surname>Dey</surname>
              <given-names>AK</given-names>
            </name>
            <name name-style="western">
              <surname>Wang</surname>
              <given-names>D</given-names>
            </name>
          </person-group>
          <article-title>Mental-LLM: leveraging large language models for mental health prediction via online text data</article-title>
          <source>Proc ACM Interact Mob Wearable Ubiquitous Technol</source>
          <year>2024</year>
          <month>03</month>
          <day>06</day>
          <volume>8</volume>
          <issue>1</issue>
          <fpage>1</fpage>
          <lpage>32</lpage>
          <pub-id pub-id-type="doi">10.1145/3643540</pub-id>
        </nlm-citation>
      </ref>
      <ref id="ref44">
        <label>44</label>
        <nlm-citation citation-type="confproc">
          <person-group person-group-type="author">
            <name name-style="western">
              <surname>Salutari</surname>
              <given-names>F</given-names>
            </name>
            <name name-style="western">
              <surname>Ramos</surname>
              <given-names>J</given-names>
            </name>
            <name name-style="western">
              <surname>Rahmani</surname>
              <given-names>HA</given-names>
            </name>
            <name name-style="western">
              <surname>Linguaglossa</surname>
              <given-names>L</given-names>
            </name>
            <name name-style="western">
              <surname>Lipani</surname>
              <given-names>A</given-names>
            </name>
          </person-group>
          <article-title>Quantifying the bias of transformer-based language models for African American English in masked language modeling</article-title>
          <source>Proceedings of the 27th Pacific-Asia Conference on Knowledge Discovery and Data Mining</source>
          <year>2023</year>
          <conf-name>PAKDD 2023</conf-name>
          <conf-date>May 25-28, 2023</conf-date>
          <conf-loc>Osaka, Japan</conf-loc>
          <pub-id pub-id-type="doi">10.1007/978-3-031-33374-3_42</pub-id>
        </nlm-citation>
      </ref>
      <ref id="ref45">
        <label>45</label>
        <nlm-citation citation-type="journal">
          <person-group person-group-type="author">
            <name name-style="western">
              <surname>Li</surname>
              <given-names>B</given-names>
            </name>
            <name name-style="western">
              <surname>Peng</surname>
              <given-names>H</given-names>
            </name>
            <name name-style="western">
              <surname>Sainju</surname>
              <given-names>R</given-names>
            </name>
            <name name-style="western">
              <surname>Yang</surname>
              <given-names>J</given-names>
            </name>
            <name name-style="western">
              <surname>Yang</surname>
              <given-names>L</given-names>
            </name>
            <name name-style="western">
              <surname>Liang</surname>
              <given-names>Y</given-names>
            </name>
            <name name-style="western">
              <surname>Jiang</surname>
              <given-names>W</given-names>
            </name>
            <name name-style="western">
              <surname>Wang</surname>
              <given-names>B</given-names>
            </name>
            <name name-style="western">
              <surname>Liu</surname>
              <given-names>H</given-names>
            </name>
            <name name-style="western">
              <surname>Ding</surname>
              <given-names>C</given-names>
            </name>
          </person-group>
          <article-title>Detecting gender bias in transformer-based models: a case study on BERT</article-title>
          <source>arXiv</source>
          <comment>Preprint posted online on October 15, 2021</comment>
          <pub-id pub-id-type="doi">10.48550/arXiv.2110.15733</pub-id>
        </nlm-citation>
      </ref>
      <ref id="ref46">
        <label>46</label>
        <nlm-citation citation-type="confproc">
          <person-group person-group-type="author">
            <name name-style="western">
              <surname>Kamboj</surname>
              <given-names>P</given-names>
            </name>
            <name name-style="western">
              <surname>Kumar</surname>
              <given-names>S</given-names>
            </name>
            <name name-style="western">
              <surname>Goyal</surname>
              <given-names>V</given-names>
            </name>
          </person-group>
          <article-title>Measuring and mitigating gender bias in contextualized word embeddings</article-title>
          <source>Proceedings of the IEEE International Conference on Blockchain and Distributed Systems Security</source>
          <year>2023</year>
          <conf-name>ICBDS 2023</conf-name>
          <conf-date>October 6-8, 2023</conf-date>
          <conf-loc>New Raipur, India</conf-loc>
          <pub-id pub-id-type="doi">10.1109/icbds58040.2023.10346586</pub-id>
        </nlm-citation>
      </ref>
      <ref id="ref47">
        <label>47</label>
        <nlm-citation citation-type="journal">
          <person-group person-group-type="author">
            <name name-style="western">
              <surname>Meade</surname>
              <given-names>N</given-names>
            </name>
            <name name-style="western">
              <surname>Poole-Dayan</surname>
              <given-names>E</given-names>
            </name>
            <name name-style="western">
              <surname>Reddy</surname>
              <given-names>S</given-names>
            </name>
          </person-group>
          <article-title>An empirical survey of the effectiveness of debiasing techniques for pre-trained language models</article-title>
          <source>arXiv</source>
          <comment>Preprint posted online on October 16, 2021</comment>
          <comment>
            <ext-link ext-link-type="uri" xlink:type="simple" xlink:href="https://arxiv.org/abs/2110.08527#:~:text=We%20experimentally%20find%20that%3A%20(1,CrowS%2DPairs%20by%20using%20debiasing"/>
          </comment>
          <pub-id pub-id-type="doi">10.18653/v1/2022.acl-long.132</pub-id>
        </nlm-citation>
      </ref>
      <ref id="ref48">
        <label>48</label>
        <nlm-citation citation-type="confproc">
          <person-group person-group-type="author">
            <name name-style="western">
              <surname>Gira</surname>
              <given-names>M</given-names>
            </name>
            <name name-style="western">
              <surname>Zhang</surname>
              <given-names>R</given-names>
            </name>
            <name name-style="western">
              <surname>Lee</surname>
              <given-names>K</given-names>
            </name>
          </person-group>
          <article-title>Debiasing pre-trained language models via efficient fine-tuning</article-title>
          <source>Proceedings of the Second Workshop on Language Technology for Equality, Diversity and Inclusion</source>
          <year>2022</year>
          <conf-name>LTEDI 2022</conf-name>
          <conf-date>May 27, 2022</conf-date>
          <conf-loc>Dublin, Ireland</conf-loc>
          <pub-id pub-id-type="doi">10.18653/v1/2022.ltedi-1.8</pub-id>
        </nlm-citation>
      </ref>
      <ref id="ref49">
        <label>49</label>
        <nlm-citation citation-type="journal">
          <person-group person-group-type="author">
            <name name-style="western">
              <surname>Kumar</surname>
              <given-names>A</given-names>
            </name>
            <name name-style="western">
              <surname>Murthy</surname>
              <given-names>SV</given-names>
            </name>
            <name name-style="western">
              <surname>Singh</surname>
              <given-names>S</given-names>
            </name>
            <name name-style="western">
              <surname>Ragupathy</surname>
              <given-names>S</given-names>
            </name>
          </person-group>
          <article-title>The ethics of interaction: mitigating security threats in LLMs</article-title>
          <source>arXiv</source>
          <comment>Preprint posted online on January 22, 2024</comment>
        </nlm-citation>
      </ref>
      <ref id="ref50">
        <label>50</label>
        <nlm-citation citation-type="journal">
          <person-group person-group-type="author">
            <name name-style="western">
              <surname>Khowaja</surname>
              <given-names>SA</given-names>
            </name>
            <name name-style="western">
              <surname>Khuwaja</surname>
              <given-names>P</given-names>
            </name>
            <name name-style="western">
              <surname>Dev</surname>
              <given-names>K</given-names>
            </name>
            <name name-style="western">
              <surname>Wang</surname>
              <given-names>W</given-names>
            </name>
            <name name-style="western">
              <surname>Nkenyereye</surname>
              <given-names>L</given-names>
            </name>
          </person-group>
          <article-title>ChatGPT needs SPADE (sustainability, PrivAcy, digital divide, and ethics) evaluation: a review</article-title>
          <source>Cogn Comput</source>
          <year>2024</year>
          <month>05</month>
          <day>05</day>
          <pub-id pub-id-type="doi">10.1007/s12559-024-10285-1</pub-id>
        </nlm-citation>
      </ref>
      <ref id="ref51">
        <label>51</label>
        <nlm-citation citation-type="journal">
          <person-group person-group-type="author">
            <name name-style="western">
              <surname>Coghlan</surname>
              <given-names>S</given-names>
            </name>
            <name name-style="western">
              <surname>Leins</surname>
              <given-names>K</given-names>
            </name>
            <name name-style="western">
              <surname>Sheldrick</surname>
              <given-names>S</given-names>
            </name>
            <name name-style="western">
              <surname>Cheong</surname>
              <given-names>M</given-names>
            </name>
            <name name-style="western">
              <surname>Gooding</surname>
              <given-names>P</given-names>
            </name>
            <name name-style="western">
              <surname>D'Alfonso</surname>
              <given-names>S</given-names>
            </name>
          </person-group>
          <article-title>To chat or bot to chat: ethical issues with using chatbots in mental health</article-title>
          <source>Digit Health</source>
          <year>2023</year>
          <month>06</month>
          <day>22</day>
          <volume>9</volume>
          <fpage>20552076231183542</fpage>
          <comment>
            <ext-link ext-link-type="uri" xlink:type="simple" xlink:href="https://journals.sagepub.com/doi/10.1177/20552076231183542?url_ver=Z39.88-2003&#38;rfr_id=ori:rid:crossref.org&#38;rfr_dat=cr_pub  0pubmed"/>
          </comment>
          <pub-id pub-id-type="doi">10.1177/20552076231183542</pub-id>
          <pub-id pub-id-type="medline">37377565</pub-id>
          <pub-id pub-id-type="pii">10.1177_20552076231183542</pub-id>
          <pub-id pub-id-type="pmcid">PMC10291862</pub-id>
        </nlm-citation>
      </ref>
      <ref id="ref52">
        <label>52</label>
        <nlm-citation citation-type="journal">
          <person-group person-group-type="author">
            <name name-style="western">
              <surname>Agrawal</surname>
              <given-names>A</given-names>
            </name>
            <name name-style="western">
              <surname>Kedia</surname>
              <given-names>N</given-names>
            </name>
            <name name-style="western">
              <surname>Panwar</surname>
              <given-names>A</given-names>
            </name>
            <name name-style="western">
              <surname>Mohan</surname>
              <given-names>J</given-names>
            </name>
            <name name-style="western">
              <surname>Kwatra</surname>
              <given-names>N</given-names>
            </name>
            <name name-style="western">
              <surname>Gulavani</surname>
              <given-names>BS</given-names>
            </name>
            <name name-style="western">
              <surname>Tumanov</surname>
              <given-names>A</given-names>
            </name>
            <name name-style="western">
              <surname>Ramjee</surname>
              <given-names>R</given-names>
            </name>
          </person-group>
          <article-title>Taming throughput-latency tradeoff in LLM inference with Sarathi-serve</article-title>
          <source>arXiv</source>
          <comment>Preprint posted online on March 4, 2024</comment>
          <comment>
            <ext-link ext-link-type="uri" xlink:type="simple" xlink:href="https://arxiv.org/abs/2403.02310"/>
          </comment>
          <pub-id pub-id-type="doi">10.48550/arXiv.2403.02310</pub-id>
        </nlm-citation>
      </ref>
      <ref id="ref53">
        <label>53</label>
        <nlm-citation citation-type="confproc">
          <person-group person-group-type="author">
            <name name-style="western">
              <surname>Santhanam</surname>
              <given-names>S</given-names>
            </name>
            <name name-style="western">
              <surname>Karduni</surname>
              <given-names>A</given-names>
            </name>
            <name name-style="western">
              <surname>Shaikh</surname>
              <given-names>S</given-names>
            </name>
          </person-group>
          <article-title>Studying the effects of cognitive biases in evaluation of conversational agents</article-title>
          <source>Proceedings of the 2020 CHI Conference on Human Factors in Computing Systems</source>
          <year>2020</year>
          <conf-name>CHI '20</conf-name>
          <conf-date>April 25-30, 2020</conf-date>
          <conf-loc>Honolulu, HI</conf-loc>
          <pub-id pub-id-type="doi">10.1145/3313831.3376318</pub-id>
        </nlm-citation>
      </ref>
      <ref id="ref54">
        <label>54</label>
        <nlm-citation citation-type="journal">
          <person-group person-group-type="author">
            <name name-style="western">
              <surname>Gnewuch</surname>
              <given-names>U</given-names>
            </name>
            <name name-style="western">
              <surname>Morana</surname>
              <given-names>S</given-names>
            </name>
            <name name-style="western">
              <surname>Adam</surname>
              <given-names>MT</given-names>
            </name>
            <name name-style="western">
              <surname>Maedche</surname>
              <given-names>A</given-names>
            </name>
          </person-group>
          <article-title>Opposing effects of response time in human–chatbot interaction</article-title>
          <source>Bus Inf Syst Eng</source>
          <year>2022</year>
          <month>05</month>
          <day>30</day>
          <volume>64</volume>
          <fpage>773</fpage>
          <lpage>91</lpage>
          <pub-id pub-id-type="doi">10.1007/s12599-022-00755-x</pub-id>
        </nlm-citation>
      </ref>
      <ref id="ref55">
        <label>55</label>
        <nlm-citation citation-type="journal">
          <person-group person-group-type="author">
            <name name-style="western">
              <surname>Prince</surname>
              <given-names>SA</given-names>
            </name>
            <name name-style="western">
              <surname>Cardilli</surname>
              <given-names>L</given-names>
            </name>
            <name name-style="western">
              <surname>Reed</surname>
              <given-names>JL</given-names>
            </name>
            <name name-style="western">
              <surname>Saunders</surname>
              <given-names>TJ</given-names>
            </name>
            <name name-style="western">
              <surname>Kite</surname>
              <given-names>C</given-names>
            </name>
            <name name-style="western">
              <surname>Douillette</surname>
              <given-names>K</given-names>
            </name>
            <name name-style="western">
              <surname>Fournier</surname>
              <given-names>K</given-names>
            </name>
            <name name-style="western">
              <surname>Buckley</surname>
              <given-names>JP</given-names>
            </name>
          </person-group>
          <article-title>A comparison of self-reported and device measured sedentary behaviour in adults: a systematic review and meta-analysis</article-title>
          <source>Int J Behav Nutr Phys Act</source>
          <year>2020</year>
          <month>03</month>
          <day>04</day>
          <volume>17</volume>
          <issue>1</issue>
          <fpage>31</fpage>
          <comment>
            <ext-link ext-link-type="uri" xlink:type="simple" xlink:href="https://ijbnpa.biomedcentral.com/articles/10.1186/s12966-020-00938-3"/>
          </comment>
          <pub-id pub-id-type="doi">10.1186/s12966-020-00938-3</pub-id>
          <pub-id pub-id-type="medline">32131845</pub-id>
          <pub-id pub-id-type="pii">10.1186/s12966-020-00938-3</pub-id>
          <pub-id pub-id-type="pmcid">PMC7055033</pub-id>
        </nlm-citation>
      </ref>
      <ref id="ref56">
        <label>56</label>
        <nlm-citation citation-type="journal">
          <person-group person-group-type="author">
            <name name-style="western">
              <surname>Iyer</surname>
              <given-names>R</given-names>
            </name>
            <name name-style="western">
              <surname>Nedeljkovic</surname>
              <given-names>M</given-names>
            </name>
            <name name-style="western">
              <surname>Meyer</surname>
              <given-names>D</given-names>
            </name>
          </person-group>
          <article-title>Using voice biomarkers to classify suicide risk in adult telehealth callers: retrospective observational study</article-title>
          <source>JMIR Ment Health</source>
          <year>2022</year>
          <month>08</month>
          <day>15</day>
          <volume>9</volume>
          <issue>8</issue>
          <fpage>e39807</fpage>
          <comment>
            <ext-link ext-link-type="uri" xlink:type="simple" xlink:href="https://mental.jmir.org/2022/8/e39807/"/>
          </comment>
          <pub-id pub-id-type="doi">10.2196/39807</pub-id>
          <pub-id pub-id-type="medline">35969444</pub-id>
          <pub-id pub-id-type="pii">v9i8e39807</pub-id>
          <pub-id pub-id-type="pmcid">PMC9425169</pub-id>
        </nlm-citation>
      </ref>
      <ref id="ref57">
        <label>57</label>
        <nlm-citation citation-type="journal">
          <person-group person-group-type="author">
            <name name-style="western">
              <surname>Iyer</surname>
              <given-names>R</given-names>
            </name>
            <name name-style="western">
              <surname>Nedeljkovic</surname>
              <given-names>M</given-names>
            </name>
            <name name-style="western">
              <surname>Meyer</surname>
              <given-names>D</given-names>
            </name>
          </person-group>
          <article-title>Using vocal characteristics to classify psychological distress in adult helpline callers: retrospective observational study</article-title>
          <source>JMIR Form Res</source>
          <year>2022</year>
          <month>12</month>
          <day>19</day>
          <volume>6</volume>
          <issue>12</issue>
          <fpage>e42249</fpage>
          <comment>
            <ext-link ext-link-type="uri" xlink:type="simple" xlink:href="https://formative.jmir.org/2022/12/e42249/"/>
          </comment>
          <pub-id pub-id-type="doi">10.2196/42249</pub-id>
          <pub-id pub-id-type="medline">36534456</pub-id>
          <pub-id pub-id-type="pii">v6i12e42249</pub-id>
          <pub-id pub-id-type="pmcid">PMC9811648</pub-id>
        </nlm-citation>
      </ref>
      <ref id="ref58">
        <label>58</label>
        <nlm-citation citation-type="confproc">
          <person-group person-group-type="author">
            <name name-style="western">
              <surname>Schaeffler</surname>
              <given-names>F</given-names>
            </name>
            <name name-style="western">
              <surname>Jannetts</surname>
              <given-names>S</given-names>
            </name>
            <name name-style="western">
              <surname>Beck</surname>
              <given-names>J</given-names>
            </name>
          </person-group>
          <article-title>Reliability of clinical voice parameters captured with smartphones — measurements of added noise and spectral tilt</article-title>
          <source>Proceedings of the 20th Annual Conference of the International Speech Communication Association</source>
          <year>2019</year>
          <conf-name>INTERSPEECH 2019</conf-name>
          <conf-date>September 15-19, 2019</conf-date>
          <conf-loc>Graz, Austria</conf-loc>
          <pub-id pub-id-type="doi">10.21437/interspeech.2019-2910</pub-id>
        </nlm-citation>
      </ref>
      <ref id="ref59">
        <label>59</label>
        <nlm-citation citation-type="journal">
          <person-group person-group-type="author">
            <name name-style="western">
              <surname>Laukka</surname>
              <given-names>P</given-names>
            </name>
            <name name-style="western">
              <surname>Elfenbein</surname>
              <given-names>HA</given-names>
            </name>
            <name name-style="western">
              <surname>Thingujam</surname>
              <given-names>NS</given-names>
            </name>
            <name name-style="western">
              <surname>Rockstuhl</surname>
              <given-names>T</given-names>
            </name>
            <name name-style="western">
              <surname>Iraki</surname>
              <given-names>FK</given-names>
            </name>
            <name name-style="western">
              <surname>Chui</surname>
              <given-names>W</given-names>
            </name>
            <name name-style="western">
              <surname>Althoff</surname>
              <given-names>J</given-names>
            </name>
          </person-group>
          <article-title>The expression and recognition of emotions in the voice across five nations: a lens model analysis based on acoustic features</article-title>
          <source>J Pers Soc Psychol</source>
          <year>2016</year>
          <month>11</month>
          <volume>111</volume>
          <issue>5</issue>
          <fpage>686</fpage>
          <lpage>705</lpage>
          <pub-id pub-id-type="doi">10.1037/pspi0000066</pub-id>
          <pub-id pub-id-type="medline">27537275</pub-id>
          <pub-id pub-id-type="pii">2016-40098-001</pub-id>
        </nlm-citation>
      </ref>
      <ref id="ref60">
        <label>60</label>
        <nlm-citation citation-type="journal">
          <person-group person-group-type="author">
            <name name-style="western">
              <surname>Markova</surname>
              <given-names>D</given-names>
            </name>
            <name name-style="western">
              <surname>Richer</surname>
              <given-names>L</given-names>
            </name>
            <name name-style="western">
              <surname>Pangelinan</surname>
              <given-names>M</given-names>
            </name>
            <name name-style="western">
              <surname>Schwartz</surname>
              <given-names>DH</given-names>
            </name>
            <name name-style="western">
              <surname>Leonard</surname>
              <given-names>G</given-names>
            </name>
            <name name-style="western">
              <surname>Perron</surname>
              <given-names>M</given-names>
            </name>
            <name name-style="western">
              <surname>Pike</surname>
              <given-names>GB</given-names>
            </name>
            <name name-style="western">
              <surname>Veillette</surname>
              <given-names>S</given-names>
            </name>
            <name name-style="western">
              <surname>Chakravarty</surname>
              <given-names>MM</given-names>
            </name>
            <name name-style="western">
              <surname>Pausova</surname>
              <given-names>Z</given-names>
            </name>
            <name name-style="western">
              <surname>Paus</surname>
              <given-names>T</given-names>
            </name>
          </person-group>
          <article-title>Age- and sex-related variations in vocal-tract morphology and voice acoustics during adolescence</article-title>
          <source>Horm Behav</source>
          <year>2016</year>
          <month>05</month>
          <volume>81</volume>
          <fpage>84</fpage>
          <lpage>96</lpage>
          <pub-id pub-id-type="doi">10.1016/j.yhbeh.2016.03.001</pub-id>
          <pub-id pub-id-type="medline">27062936</pub-id>
          <pub-id pub-id-type="pii">S0018-506X(16)30127-1</pub-id>
        </nlm-citation>
      </ref>
      <ref id="ref61">
        <label>61</label>
        <nlm-citation citation-type="journal">
          <person-group person-group-type="author">
            <name name-style="western">
              <surname>Israelsson</surname>
              <given-names>A</given-names>
            </name>
            <name name-style="western">
              <surname>Seiger</surname>
              <given-names>A</given-names>
            </name>
            <name name-style="western">
              <surname>Laukka</surname>
              <given-names>P</given-names>
            </name>
          </person-group>
          <article-title>Blended emotions can be accurately recognized from dynamic facial and vocal expressions</article-title>
          <source>J Nonverbal Behav</source>
          <year>2023</year>
          <month>05</month>
          <day>17</day>
          <volume>47</volume>
          <issue>3</issue>
          <fpage>267</fpage>
          <lpage>84</lpage>
          <pub-id pub-id-type="doi">10.1007/s10919-023-00426-9</pub-id>
        </nlm-citation>
      </ref>
      <ref id="ref62">
        <label>62</label>
        <nlm-citation citation-type="journal">
          <person-group person-group-type="author">
            <name name-style="western">
              <surname>Busquet</surname>
              <given-names>F</given-names>
            </name>
            <name name-style="western">
              <surname>Efthymiou</surname>
              <given-names>F</given-names>
            </name>
            <name name-style="western">
              <surname>Hildebrand</surname>
              <given-names>C</given-names>
            </name>
          </person-group>
          <article-title>Voice analytics in the wild: validity and predictive accuracy of common audio-recording devices</article-title>
          <source>Behav Res Methods</source>
          <year>2024</year>
          <month>03</month>
          <day>30</day>
          <volume>56</volume>
          <issue>3</issue>
          <fpage>2114</fpage>
          <lpage>34</lpage>
          <comment>
            <ext-link ext-link-type="uri" xlink:type="simple" xlink:href="https://europepmc.org/abstract/MED/37253958"/>
          </comment>
          <pub-id pub-id-type="doi">10.3758/s13428-023-02139-9</pub-id>
          <pub-id pub-id-type="medline">37253958</pub-id>
          <pub-id pub-id-type="pii">10.3758/s13428-023-02139-9</pub-id>
          <pub-id pub-id-type="pmcid">PMC10228884</pub-id>
        </nlm-citation>
      </ref>
      <ref id="ref63">
        <label>63</label>
        <nlm-citation citation-type="confproc">
          <person-group person-group-type="author">
            <name name-style="western">
              <surname>Padmapriya</surname>
              <given-names>J</given-names>
            </name>
            <name name-style="western">
              <surname>Sasilatha</surname>
              <given-names>T</given-names>
            </name>
            <name name-style="western">
              <surname>R</surname>
              <given-names>K</given-names>
            </name>
            <name name-style="western">
              <surname>Aagash</surname>
              <given-names>G</given-names>
            </name>
            <name name-style="western">
              <surname>Bharathi</surname>
              <given-names>V</given-names>
            </name>
          </person-group>
          <article-title>Voice extraction from background noise using filter bank analysis for voice communication applications</article-title>
          <source>Proceedings of the Third International Conference on Intelligent Communication Technologies and Virtual Mobile Networks</source>
          <year>2021</year>
          <conf-name>ICICV 2021</conf-name>
          <conf-date>February 4-6, 2021</conf-date>
          <conf-loc>Tirunelveli, India</conf-loc>
          <pub-id pub-id-type="doi">10.1109/icicv50876.2021.9388453</pub-id>
        </nlm-citation>
      </ref>
    </ref-list>
  </back>
</article>
