<?xml version="1.0" encoding="UTF-8"?>
<!DOCTYPE article PUBLIC "-//NLM//DTD Journal Publishing DTD v2.0 20040830//EN" "http://dtd.nlm.nih.gov/publishing/2.0/journalpublishing.dtd">
<?covid-19-tdm?>
<article xmlns:xlink="http://www.w3.org/1999/xlink" article-type="research-article" dtd-version="2.0">
  <front>
    <journal-meta>
      <journal-id journal-id-type="publisher-id">JMH</journal-id>
      <journal-id journal-id-type="nlm-ta">JMIR Ment Health</journal-id>
      <journal-title>JMIR Mental Health</journal-title>
      <issn pub-type="epub">2368-7959</issn>
      <publisher>
        <publisher-name>JMIR Publications</publisher-name>
        <publisher-loc>Toronto, Canada</publisher-loc>
      </publisher>
    </journal-meta>
    <article-meta>
      <article-id pub-id-type="publisher-id">v8i4e25097</article-id>
      <article-id pub-id-type="pmid">33877051</article-id>
      <article-id pub-id-type="doi">10.2196/25097</article-id>
      <article-categories>
        <subj-group subj-group-type="heading">
          <subject>Original Paper</subject>
        </subj-group>
        <subj-group subj-group-type="article-type">
          <subject>Original Paper</subject>
        </subj-group>
      </article-categories>
      <title-group>
        <article-title>Learning the Mental Health Impact of COVID-19 in the United States With Explainable Artificial Intelligence: Observational Study</article-title>
      </title-group>
      <contrib-group>
        <contrib contrib-type="editor">
          <name>
            <surname>Torous</surname>
            <given-names>John</given-names>
          </name>
        </contrib>
      </contrib-group>
      <contrib-group>
        <contrib contrib-type="reviewer">
          <name>
            <surname>Rau</surname>
            <given-names>Hsiao-Hsien</given-names>
          </name>
        </contrib>
        <contrib contrib-type="reviewer">
          <name>
            <surname>Gabashvili</surname>
            <given-names>Irene</given-names>
          </name>
        </contrib>
      </contrib-group>
      <contrib-group>
        <contrib id="contrib1" contrib-type="author" equal-contrib="yes">
          <name name-style="western">
            <surname>Jha</surname>
            <given-names>Indra Prakash</given-names>
          </name>
          <degrees>MCA</degrees>
          <xref rid="aff1" ref-type="aff">1</xref>
          <ext-link ext-link-type="orcid">https://orcid.org/0000-0003-4780-5853</ext-link>
        </contrib>
        <contrib id="contrib2" contrib-type="author" equal-contrib="yes">
          <name name-style="western">
            <surname>Awasthi</surname>
            <given-names>Raghav</given-names>
          </name>
          <degrees>MSc</degrees>
          <xref rid="aff1" ref-type="aff">1</xref>
          <ext-link ext-link-type="orcid">https://orcid.org/0000-0002-6643-4333</ext-link>
        </contrib>
        <contrib id="contrib3" contrib-type="author">
          <name name-style="western">
            <surname>Kumar</surname>
            <given-names>Ajit</given-names>
          </name>
          <degrees>MBA</degrees>
          <xref rid="aff2" ref-type="aff">2</xref>
          <ext-link ext-link-type="orcid">https://orcid.org/0000-0001-9494-2629</ext-link>
        </contrib>
        <contrib id="contrib4" contrib-type="author">
          <name name-style="western">
            <surname>Kumar</surname>
            <given-names>Vibhor</given-names>
          </name>
          <degrees>PhD</degrees>
          <xref rid="aff1" ref-type="aff">1</xref>
          <ext-link ext-link-type="orcid">https://orcid.org/0000-0002-9523-9422</ext-link>
        </contrib>
        <contrib id="contrib5" contrib-type="author" corresp="yes">
          <name name-style="western">
            <surname>Sethi</surname>
            <given-names>Tavpritesh</given-names>
          </name>
          <degrees>PhD</degrees>
          <xref rid="aff1" ref-type="aff">1</xref>
          <address>
            <institution>Indraprastha Institute of Information Technology</institution>
            <addr-line>Room 309, R and D Building</addr-line>
            <addr-line>IIIT Campus, Okhla Phase 3</addr-line>
            <addr-line>New Delhi</addr-line>
            <country>India</country>
            <phone>91 01126907533</phone>
            <email>tavpriteshsethi@iiitd.ac.in</email>
          </address>
          <ext-link ext-link-type="orcid">https://orcid.org/0000-0002-4776-7941</ext-link>
        </contrib>
      </contrib-group>
      <aff id="aff1">
        <label>1</label>
        <institution>Indraprastha Institute of Information Technology</institution>
        <addr-line>New Delhi</addr-line>
        <country>India</country>
      </aff>
      <aff id="aff2">
        <label>2</label>
        <institution>Adobe</institution>
        <addr-line>Noida</addr-line>
        <country>India</country>
      </aff>
      <author-notes>
        <corresp>Corresponding Author: Tavpritesh Sethi <email>tavpriteshsethi@iiitd.ac.in</email></corresp>
      </author-notes>
      <pub-date pub-type="collection">
        <month>4</month>
        <year>2021</year>
      </pub-date>
      <pub-date pub-type="epub">
        <day>20</day>
        <month>4</month>
        <year>2021</year>
      </pub-date>
      <volume>8</volume>
      <issue>4</issue>
      <elocation-id>e25097</elocation-id>
      <history>
        <date date-type="received">
          <day>17</day>
          <month>10</month>
          <year>2020</year>
        </date>
        <date date-type="rev-request">
          <day>20</day>
          <month>11</month>
          <year>2020</year>
        </date>
        <date date-type="rev-recd">
          <day>10</day>
          <month>12</month>
          <year>2020</year>
        </date>
        <date date-type="accepted">
          <day>3</day>
          <month>2</month>
          <year>2021</year>
        </date>
      </history>
      <copyright-statement>©Indra Prakash Jha, Raghav Awasthi, Ajit Kumar, Vibhor Kumar, Tavpritesh Sethi. Originally published in JMIR Mental Health (https://mental.jmir.org), 20.04.2021.</copyright-statement>
      <copyright-year>2021</copyright-year>
      <license license-type="open-access" xlink:href="https://creativecommons.org/licenses/by/4.0/">
        <p>This is an open-access article distributed under the terms of the Creative Commons Attribution License (https://creativecommons.org/licenses/by/4.0/), which permits unrestricted use, distribution, and reproduction in any medium, provided the original work, first published in JMIR Mental Health, is properly cited. The complete bibliographic information, a link to the original publication on http://mental.jmir.org/, as well as this copyright and license information must be included.</p>
      </license>
      <self-uri xlink:href="https://mental.jmir.org/2021/4/e25097" xlink:type="simple"/>
      <abstract>
        <sec sec-type="background">
          <title>Background</title>
          <p>The COVID-19 pandemic has affected the health, economic, and social fabric of many nations worldwide. Identification of individual-level susceptibility factors may help people in identifying and managing their emotional, psychological, and social well-being.</p>
        </sec>
        <sec sec-type="objective">
          <title>Objective</title>
          <p>This study is focused on learning a ranked list of factors that could indicate a predisposition to a mental disorder during the COVID-19 pandemic.</p>
        </sec>
        <sec sec-type="methods">
          <title>Methods</title>
          <p>In this study, we have used a survey of 17,764  adults in the United States from different age groups, genders, and socioeconomic statuses. Through initial statistical analysis and Bayesian network inference, we have identified key factors affecting mental health during the COVID-19 pandemic. Integrating Bayesian networks with classical machine learning approaches led to effective modeling of the level of mental health prevalence.</p>
        </sec>
        <sec sec-type="results">
          <title>Results</title>
          <p>Overall, females were more stressed than males, and people in the age group 18-29 years were more vulnerable to anxiety than other age groups. Using the Bayesian network model, we found that people with a chronic mental illness were more prone to mental disorders during the COVID-19 pandemic. The new realities of working from home; homeschooling; and lack of communication with family, friends, and neighbors induces mental pressure. Financial assistance from social security helps in reducing mental stress during the COVID-19–generated economic crises. Finally, using supervised machine learning models, we predicted the most mentally vulnerable people with ~80% accuracy.</p>
        </sec>
        <sec sec-type="conclusions">
          <title>Conclusions</title>
          <p>Multiple factors such as social isolation, digital communication, and working and schooling from home were identified as factors of mental illness during the COVID-19 pandemic. Regular in-person communication with friends and family, a healthy social life, and social security were key factors, and taking care of people with a history of mental disease appears to be even more important during this time.</p>
        </sec>
      </abstract>
      <kwd-group>
        <kwd>COVID-19</kwd>
        <kwd>mental health</kwd>
        <kwd>Bayesian network</kwd>
        <kwd>machine learning</kwd>
        <kwd>artificial intelligence</kwd>
        <kwd>disorder</kwd>
        <kwd>susceptibility</kwd>
        <kwd>well-being</kwd>
        <kwd>explainable artificial intelligence</kwd>
      </kwd-group>
    </article-meta>
  </front>
  <body>
    <sec sec-type="introduction">
      <title>Introduction</title>
      <p>After 7 months of initial reporting, the COVID-19 pandemic continues worldwide. The mental health consequences of the COVID-19 pandemic have been substantial. More than half a million lives and more than 400 million jobs have been lost [<xref ref-type="bibr" rid="ref1">1</xref>], causing a considerable degree of fear, worry, and concern. These effects are seen in the population at large and may be more pronounced among certain groups such as youth, frontline workers [<xref ref-type="bibr" rid="ref2">2</xref>], caregivers, and people with chronic medical conditions. The new normal has introduced unprecedented interventions of countrywide lockdowns that are necessary to control the spread but have led to increased social isolation. Loneliness, depression, harmful alcohol and drug use, and self-harm or suicidal behavior are also expected to rise.</p>
      <p>The Lancet Psychiatry [<xref ref-type="bibr" rid="ref3">3</xref>] recently highlighted the needs of vulnerable groups during this time, including those with severe mental illness, learning difficulties, and neurodevelopmental disorders, as well as socially excluded groups such as prisoners, the homeless, and refugees. Calls to action for engaging more early-career psychiatrists [<xref ref-type="bibr" rid="ref4">4</xref>,<xref ref-type="bibr" rid="ref5">5</xref>], using technology such as telepsychiatry, and stressing the high susceptibility of frontline medical workers themselves [<xref ref-type="bibr" rid="ref6">6</xref>] have highlighted the magnitude of the problem. Further, interventions are expected to have a gender-specific impact, with women more likely to be exposed to additional stressors related to informal care, already existing economic disparity, and school closures. Similarly, age and comorbidity status may have a direct impact on susceptibility to mental health challenges due to their relationship with COVID-19 morbidity and mortality. Indeed, it has been established that emotional distress is ubiquitous in affected populations—a finding certain to be echoed in populations affected by the COVID-19 pandemic [<xref ref-type="bibr" rid="ref7">7</xref>]. Finally, the role of social media [<xref ref-type="bibr" rid="ref8">8</xref>,<xref ref-type="bibr" rid="ref9">9</xref>] is complex, with some research indicating an association between social media exposure and a higher prevalence of mental health problems [<xref ref-type="bibr" rid="ref10">10</xref>].</p>
      <p>However, most of these effects have been studied in isolation with a lack of modeling the collective impact of such factors. This study addresses this gap through the use of Bayesian networks (BNs), an explainable artificial intelligence approach that captures the joint multivariate distribution underlying large survey data collected across the United States. We also address the gap of vulnerability prediction for mental health events such as anxiety attacks using supervised machine learning models.</p>
    </sec>
    <sec sec-type="methods">
      <title>Methods</title>
      <sec>
        <title>Data Sets</title>
        <p>We extracted the data of 17,764 adults [<xref ref-type="bibr" rid="ref11">11</xref>] from two weekly surveys (April 20-26 and May 4-10, 2020) of the US adult household population nationwide for 18 regional areas including 10 states (California, Colorado, Florida, Louisiana, Minnesota, Missouri, Montana, New York, Oregon, Texas) and 8 metropolitan statistical areas (Atlanta, Baltimore, Birmingham, Chicago, Cleveland, Columbus, Phoenix, Pittsburgh). Two rounds of data collection were available at the time of this analysis, and both rounds of data until May 25, 2020, were included in this analysis. The details of the original data are available elsewhere [<xref ref-type="bibr" rid="ref12">12</xref>]. To summarize, the data set comprised variables on physical health, mental health, insurance-related policy, economic security, and social dynamics. <xref rid="figure1" ref-type="fig">Figure 1</xref> shows the sociodemographic characteristics of respondents participating in the survey.</p>
        <fig id="figure1" position="float">
          <label>Figure 1</label>
          <caption>
            <p>Sociodemographics of respondents who participated in the survey. It can be seen that there was almost a similar representation from both genders. Age groups 25-75 years were predominantly captured in the survey. Most of the respondents had received a Bachelor's degree or above and were nearly equally distributed across geographies within the United States. HS: high school.</p>
          </caption>
          <graphic xlink:href="mental_v8i4e25097_fig1.png" alt-version="no" mimetype="image" position="float" xlink:type="simple"/>
        </fig>
      </sec>
      <sec>
        <title>Analysis</title>
        <p><xref rid="figure2" ref-type="fig">Figure 2</xref>a shows the flow diagram for the analyses conducted. The survey questions were classified into several types of indicators such as mental health, work from home, communication, COVID-19 symptoms, chronic medical conditions, behavioral aspects, insurance assistance, and many others (<xref ref-type="supplementary-material" rid="app1">Multimedia Appendix 1</xref>).</p>
        <fig id="figure2" position="float">
          <label>Figure 2</label>
          <caption>
            <p>(a) Outline of the analytical pipeline. (b) Item reliability analysis of mental health indicators revealed a high degree of internal consistency (Cronbach alpha value &#62;.70) for most of the psychological variables, thus indicating suitability for the modeling exercise. ML: machine learning.</p>
          </caption>
          <graphic xlink:href="mental_v8i4e25097_fig2.png" alt-version="no" mimetype="image" position="float" xlink:type="simple"/>
        </fig>
        <sec>
          <title>Item Reliability Analysis</title>
          <p>We constructed a model for the mental health indicators with attribute <italic>soc5a</italic> (felt nervous, anxious<italic>,</italic> or on edge), attribute <italic>soc5b</italic> (felt depressed), attribute <italic>soc5c</italic> (felt lonely), attribute <italic>soc5d</italic> (felt hopeless about the future), and <italic>soc5e</italic> (sweating, trouble breathing, pounding heart, etc in the last 7 days) as outcome variables. Hence, we first evaluated the consistency in answers to the mental health questions using an item reliability analysis. A scale for measuring the reliability of internal consistency, Cronbach alpha, was calculated using the <italic>Psych</italic> package in R (R Foundation for Statistical Computing) [<xref ref-type="bibr" rid="ref13">13</xref>].</p>
        </sec>
        <sec>
          <title>Test of Independence Among the Mental Health Indicator and Other Indicators</title>
          <p>Thereafter, a pairwise chi-square test of independence was performed to examine associations between <italic>mental health indicators</italic> and other variables, and a <italic>P</italic> value&#60;.05 was taken as the cutoff for significance.</p>
        </sec>
        <sec>
          <title>Data-Driven Bayesian Network Analysis</title>
          <p>Since mental health variables may have complex dependencies with potential confounding factors, mediation, and intercausal dependency, we extended our association analysis with data-driven BN structure learning. The structure of the learned BN was made robust through bootstrapping and ensemble averaging of edge directions. The hill climbing optimizer [<xref ref-type="bibr" rid="ref14">14</xref>] with the Akaike information criterion–based score [<xref ref-type="bibr" rid="ref15">15</xref>] was used to select the best probabilistic graphical model that explained the data. Bootstrapped learning and majority voting over 101 BNs were done. Exact inference using the belief propagation algorithm [<xref ref-type="bibr" rid="ref16">16</xref>] was learned to quantify the strength of learned associations. The analysis was performed in R using the package <italic>wiseR</italic> [<xref ref-type="bibr" rid="ref17">17</xref>].</p>
        </sec>
        <sec>
          <title>Mental Health Prediction Using Supervised Machine Learning</title>
          <p>Next, the <italic>Markov blanket</italic> [<xref ref-type="bibr" rid="ref18">18</xref>] of <italic>mental health indicators</italic> was extracted to select features that may predict responses to the <italic>mental health indicators</italic>. Data were partitioned into training (80%) and testing (20%) sets, and the class imbalance was corrected using the synthetic minority oversampling technique [<xref ref-type="bibr" rid="ref19">19</xref>]. Different supervised machine learning models—random forest (RF), support vector machine (SVM), logistic regression, naive Bayes—were learned for predicting the response to mental health indicators using the Scikit-learn library [<xref ref-type="bibr" rid="ref20">20</xref>] in Python.</p>
        </sec>
      </sec>
    </sec>
    <sec sec-type="results">
      <title>Results</title>
      <sec>
        <title>Item Reliability Analysis</title>
        <p>Attribute <italic>soc5a</italic> (felt nervous, anxious<italic>,</italic> or on edge), attribute <italic>soc5b</italic> (felt depressed), attribute <italic>soc5c</italic> (felt lonely), and attribute <italic>soc5d</italic> (felt hopeless about the future) achieved a Cronbach alpha approximating .8 (<xref rid="figure2" ref-type="fig">Figure 2</xref>b), thus confirming their internal consistency and suitability for modeling.</p>
        <sec>
          <title>Gender- and Age-Related Variation in Mental Health Indicators</title>
          <p>Gender- and age-specific difference was observed in attribute <italic>soc5a</italic>, with females having a higher incidence than males (two proportion <italic>z</italic> test, <italic>P</italic>&#60;.001; <xref rid="figure3" ref-type="fig">Figure 3</xref>a) and young adults in the 18-29 years age group having higher incidence than other age groups (<italic>P</italic>&#60;.001; <xref rid="figure3" ref-type="fig">Figure 3</xref>b). The age group 18-29 years in both genders was most vulnerable to mental stress for more than 5 days in a week, thus indicating that COVID-19 may have disproportionately affected the mental health of youth due to a variety of factors.</p>
          <fig id="figure3" position="float">
            <label>Figure 3</label>
            <caption>
              <p>(a) Genderwise and (b) agewise distribution of mental issues (attribute <italic>soc5a</italic>) variable. Significance was tested using two proportion z test and chi-square test, respectively, showing a higher prevalence of mental issues among youth and in women.</p>
            </caption>
            <graphic xlink:href="mental_v8i4e25097_fig3.png" alt-version="no" mimetype="image" position="float" xlink:type="simple"/>
          </fig>
        </sec>
      </sec>
      <sec>
        <title>Associations of Anxiety in the United States</title>
        <p>A chi-square test revealed many significant associations of the mental health variables (<xref ref-type="supplementary-material" rid="app2">Multimedia Appendix 2</xref>). However, this analysis does not account for potential confounding or <italic>explaining away</italic> effects.</p>
      </sec>
      <sec>
        <title>Data-Driven Bayesian Network Analysis</title>
        <p>Hence, a data-driven BN structure learning exercise was carried out and revealed interesting findings. From the learned structure, attribute <italic>soc5a</italic> (felt nervous, anxious, or on edge in the last 7 days) was found to be the parent variable for other mental health indicators in almost 100% of the bootstrapped networks, represented as the strength of the edges (<xref rid="figure4" ref-type="fig">Figure 4</xref>b). Being a driver variable in the structure, attribute <italic>soc5a</italic> was taken as the primary dependent variable for downstream modeling analysis.</p>
        <fig id="figure4" position="float">
          <label>Figure 4</label>
          <caption>
            <p>(a) Consensus structure learned through 101 bootstrapped samples. Hill climbing search along with Bayesian information criterion was used to learn the structures and connections having edge strength and direction strength more than 90% are shown. The color of the edges represents the proportion of networks in which that edge was present in the 101 bootstrapped samples, an indicator of confidence; (b) attribute <italic>soc5a</italic> was found to be the parent node of all other mental health variables, therefore, leading to our choice of this variable as the primary dependent variable. PGM: probabilistic graphical model.</p>
          </caption>
          <graphic xlink:href="mental_v8i4e25097_fig4.png" alt-version="no" mimetype="image" position="float" xlink:type="simple"/>
        </fig>
        <sec>
          <title>Impact of Social Life and Work-Related Stressors</title>
          <p>Our analysis using network inference via the exact inference algorithm showed a clear impact of in-person social communication on the reduction of anxiety levels. A strong (&#62;5% with CI ~1% on both sides) and (&#62;6.5% with CI ~1.5% on both sides) monotonic increase between control of anxiety and frequency of speaking with neighbors (attribute <italic>soc2a</italic>, attribute <italic>soc2b</italic>) were observed. This effect was weaker (~1.5% with a wide confidence interval) with digital communication with friends and family conducted over phone, text, email, or other internet media (attribute <italic>soc3a</italic>, attribute <italic>soc3b</italic>). This finding underscores the importance of social communication while maintaining the appropriate measures such as masks and social distancing to maintain mental health during such isolating times. We also observed that the presence of kids in the house reduces the probability of depression by &#62;11% with CI ~2% on both sides. Furthermore, the exact inference upon the network revealed an increase in the conditional probability of anxiety (attribute <italic>soc5a</italic>) arising from canceled or postponed work (&#62;4% with CI ~1.4%), canceled or postponed school (7% with CI ~1.5%), working from home (&#62;5% with CI ~1.3%), and studying from home (&#62;7% with CI ~1.8%). Interestingly, although 83% of all volunteers chose to wear the mask, 77% avoided restaurants, and 83% avoided public and crowded places, these measures were not found to be associated with a significant change in anxiety levels as inferred from our model. These inferences are summarized in <xref rid="figure5" ref-type="fig">Figure 5</xref>.</p>
          <fig id="figure5" position="float">
            <label>Figure 5</label>
            <caption>
              <p>Inferences from the Bayesian network. The difference in inferred probability was calculated after conditioning the independent variables. A positive association implies a mental stress–inducing factor, whereas a negative association implies a mental stress reduction factor. The red circle shows the mean value, with green and blue showing confidence intervals. COPD: chronic obstructive pulmonary disease; SNAP: Supplemental Nutrition Assistance Program; TANF: Temporary Assistance for Needy Families.</p>
            </caption>
            <graphic xlink:href="mental_v8i4e25097_fig5.png" alt-version="no" mimetype="image" position="float" xlink:type="simple"/>
          </fig>
        </sec>
        <sec>
          <title>Impact of Symptoms and Comorbidities</title>
          <p>We also investigated the relationship between mental stress and COVID-19 symptoms indicators. The World Health Organization recommends contacting health service providers if any COVID-19 symptoms (attributes <italic>phys1a</italic> to <italic>phys1q</italic>) are experienced within the last 7 days. Our network did not indicate any significant impact of these responses on mental health (attribute <italic>soc5a</italic>), the conditional probability of which remained unchanged (62.2%) across the responses. Although medical conditions (attributes <italic>phys3a</italic> to <italic>phys3m</italic>) are known to increase the risk of serious illness from COVID-19, our model showed that having cancer (attribute <italic>phys3k</italic>) and hypertension (attribute <italic>phys3b</italic>) had a reverse impact on anxiety levels. Those with cancer had approximately 8.3% (with ~2% CI) higher conditional probability of having less than 1 anxiety-ridden day in a week (&#62;7% effect for hypertension with CI ~1.5%). Additionally, cystic fibrosis (attribute <italic>phys3i</italic>) and liver disease (attribute <italic>phys3j</italic>) had wide confidence intervals with nonsignificant differences in mean values (<xref rid="figure5" ref-type="fig">Figure 5</xref>).</p>
        </sec>
        <sec>
          <title>Impact of Economic Factors</title>
          <p>Receiving income assistance through Social Security improved the conditional probability of less than 1 day of anxiety in a week by 10.4% (with CI ~1.5%) as compared with the segment of people who did not apply or receive it. Just applying for income assistance led to a 4% improvement (<xref rid="figure5" ref-type="fig">Figure 5</xref>). Supplemental Social Security (~5.5% with CI ~4%) and health insurance (~5% with CI ~2%) also led to similar results.</p>
          <p>In addition to this, older adults (&#62;60 years) found health insurance more relaxing than younger people. COVID-19 has also severely affected the financial condition of individuals, which may also lead to mental stress.</p>
        </sec>
      </sec>
      <sec>
        <title>Predictive Modeling for Susceptibility to Anxiety Attacks</title>
        <p>Our supervised modeling approach used the Markov blanket of the attribute <italic>soc5a</italic> variable, that is age (attribute <italic>age4),</italic> physical symptoms in the last 7 days (attribute <italic>phys7_4</italic>)<italic>,</italic> staying at home (attribute <italic>phys2_18</italic>), and prior clinical diagnosis of any mental health condition (attribute <italic>phys3h</italic>) as predictors.</p>
        <p>The following three prediction scenarios were considered:</p>
        <list list-type="order">
          <list-item>
            <p>Mental issues <italic>less than 1 day</italic> in a week (class 1) versus mental issues <italic>more than 1 day</italic> in a week (class 0)</p>
          </list-item>
          <list-item>
            <p>Mental issues <italic>less than 1 day</italic> in a week (class 1) versus mental issues <italic>more than 1 day</italic> in a week (class 0)</p>
          </list-item>
          <list-item>
            <p>Mental issues <italic>less than 1 day</italic> in a week (class 1) versus mental issues <italic>more than 1 day</italic> in a week (class 0)</p>
          </list-item>
        </list>
        <p>RF models achieved the best performance in comparison with SVM, logistic regression, and naive Bayes models on the basis of standard model performance indicators (accuracy, sensitivity, specificity, area under the receiver operating characteristic curve; summarized in <xref ref-type="table" rid="table1">Table 1</xref>). We observed a decay (accuracy from 0.80 to 0.64; <xref ref-type="table" rid="table1">Table 1</xref>) in model predictability as we moved from high risk of depression (case 3) to low risk of depression (case 1; <xref ref-type="table" rid="table1">Table 1</xref>). Such a trend was visible with all four machine learning techniques we used.</p>
        <table-wrap position="float" id="table1">
          <label>Table 1</label>
          <caption>
            <p>Model performance indicators of the supervised model for prediction of stress.</p>
          </caption>
          <table width="1000" cellpadding="5" cellspacing="0" border="1" rules="groups" frame="hsides">
            <col width="30"/>
            <col width="220"/>
            <col width="0"/>
            <col width="160"/>
            <col width="0"/>
            <col width="240"/>
            <col width="0"/>
            <col width="150"/>
            <col width="0"/>
            <col width="200"/>
            <thead>
              <tr valign="top">
                <td colspan="3">Scenarios</td>
                <td colspan="2">Random forest</td>
                <td colspan="2">Support vector machine</td>
                <td colspan="2">Naive Bayes</td>
                <td>Logistic regression</td>
              </tr>
            </thead>
            <tbody>
              <tr valign="top">
                <td colspan="10">
                  <bold>Mental issues less than 1 day in a week (class 1) vs mental issues more than 5 days in a week (class 0)</bold>
                </td>
              </tr>
              <tr valign="top">
                <td>
                  <break/>
                </td>
                <td>Accuracy (±CI)</td>
                <td colspan="2">0.80 (0.016)</td>
                <td colspan="2">0.80 (0.016)</td>
                <td colspan="2">0.77 (0.017)</td>
                <td colspan="2">0.77 (0.017)</td>
              </tr>
              <tr valign="top">
                <td>
                  <break/>
                </td>
                <td>Sensitivity (±CI)</td>
                <td colspan="2">0.59 (0.063)</td>
                <td colspan="2">0.56 (0.063)</td>
                <td colspan="2">0.59 (0.063)</td>
                <td colspan="2">0.59 (0.063)</td>
              </tr>
              <tr valign="top">
                <td>
                  <break/>
                </td>
                <td>Specificity (±CI)</td>
                <td colspan="2">0.82 (0.016)</td>
                <td colspan="2">0.82 (0.016)</td>
                <td colspan="2">0.79 (0.017)</td>
                <td colspan="2">0.78 (0.017)</td>
              </tr>
              <tr valign="top">
                <td>
                  <break/>
                </td>
                <td>AUROC<sup>a</sup> (±CI)</td>
                <td colspan="2">0.71 (0.026)</td>
                <td colspan="2">0.69 (0.026)</td>
                <td colspan="2">0.69 (0.025)</td>
                <td colspan="2">0.68 (0.025)</td>
              </tr>
              <tr valign="top">
                <td colspan="10">
                  <bold>Mental issues less than 1 day in a week (class 1) vs mental issues more than 3 days in a week (class 0)</bold>
                </td>
              </tr>
              <tr valign="top">
                <td>
                  <break/>
                </td>
                <td>Accuracy (±CI)</td>
                <td colspan="2">0.72 (0.018)</td>
                <td colspan="2">0.72 (0.018)</td>
                <td colspan="2">0.74 (0.017)</td>
                <td colspan="2">0.73 (0.018)</td>
              </tr>
              <tr valign="top">
                <td>
                  <break/>
                </td>
                <td>Sensitivity (±CI)</td>
                <td colspan="2">0.6 (0.041)</td>
                <td colspan="2">0.6 (0.041)</td>
                <td colspan="2">0.56 (0.041)</td>
                <td colspan="2">0.57 (0.041)</td>
              </tr>
              <tr valign="top">
                <td>
                  <break/>
                </td>
                <td>Specificity (±CI)</td>
                <td colspan="2">0.75 (0.018)</td>
                <td colspan="2">0.75 (0.018)</td>
                <td colspan="2">0.78 (0.017)</td>
                <td colspan="2">0.76 (0.018)</td>
              </tr>
              <tr valign="top">
                <td>
                  <break/>
                </td>
                <td>AUROC (±CI)</td>
                <td colspan="2">0.68 (0.022)</td>
                <td colspan="2">0.67 (0.022)</td>
                <td colspan="2">0.67 (0.022)</td>
                <td colspan="2">0.67 (0.022)</td>
              </tr>
              <tr valign="top">
                <td colspan="10">
                  <bold>Mental issues less than 1 day in a week (class 1) vs mental issues more than 1 day in a week (class 0)</bold>
                </td>
              </tr>
              <tr valign="top">
                <td>
                  <break/>
                </td>
                <td>Accuracy (±CI)</td>
                <td colspan="2">0.66 (0.019)</td>
                <td colspan="2">0.66 (0.019)</td>
                <td colspan="2">0.65 (0.019)</td>
                <td colspan="2">0.62 (0.019)</td>
              </tr>
              <tr valign="top">
                <td>
                  <break/>
                </td>
                <td>Sensitivity (±CI)</td>
                <td colspan="2">0.48 (0.027)</td>
                <td colspan="2">0.49 (0.027)</td>
                <td colspan="2">0.45 (0.026)</td>
                <td colspan="2">0.61 (0.026)</td>
              </tr>
              <tr valign="top">
                <td>
                  <break/>
                </td>
                <td>Specificity (±CI)</td>
                <td colspan="2">0.77 (0.018)</td>
                <td colspan="2">0.76 (0.018)</td>
                <td colspan="2">0.77 (0.018)</td>
                <td colspan="2">0.64 (0.020)</td>
              </tr>
              <tr valign="top">
                <td>
                  <break/>
                </td>
                <td>AUROC (±CI)</td>
                <td colspan="2">0.62 (0.019)</td>
                <td colspan="2">0.62 (0.019)</td>
                <td colspan="2">0.61 (0.020)</td>
                <td colspan="2">0.62 (0.018)</td>
              </tr>
            </tbody>
          </table>
          <table-wrap-foot>
            <fn id="table1fn1">
              <p><sup>a</sup>AUROC: area under the receiver operating characteristic curve.</p>
            </fn>
          </table-wrap-foot>
        </table-wrap>
      </sec>
    </sec>
    <sec sec-type="discussion">
      <title>Discussion</title>
      <p>Mental health is a public health concern. Mood disorders and suicide-related outcomes have increased substantially over the last decade among all age groups and genders [<xref ref-type="bibr" rid="ref21">21</xref>,<xref ref-type="bibr" rid="ref22">22</xref>]. The rapid spread of COVID-19 forced governments worldwide to close public gathering places, schools, colleges, restaurants, and industries. Social isolation, digital communication, and working and schooling from home have become the new normal, and many jobs have been lost. Collectively, this has triggered a high level of anxiety, stress, and depression globally. We did not find studies that have used models to not only predict but also explain the subtle effects of life situations on mental health. An explainable probabilistic graphical modeling approach with bootstraps and exact inference allowed us to capture many of these effects in a robust manner. Our study revealed that individuals with a prior diagnosis of any mental illness are the most vulnerable for mental illness during the COVID-19 pandemic, which recommends building national-level policies to regularly track their mental status and treat them accordingly. Most importantly, our results reiterate the economic underpinnings of a collective mental health response. Income assistance via Social Security or Supplemental Social Security had a demonstrable effect on the alleviation of anxiety as inferred from our model, which provides the first scientific evidence, to the best of our knowledge, proving the utility of such efforts. The extent of such measures’ effect may be captured in such modeling studies conducted in various parts of the world, with widely varying assistance structures during this time.</p>
      <p>Our findings from the United States can also stimulate further cultural and social research in other geographies with similar or different social structures. For example, the effects of in-person communication, as opposed to digital connectedness, may be different in countries where community living and joint families are still commonplace, such as India. Digital connectedness was not as effective as talking to a neighbor, at least in the United States, highlighting that these have fundamentally different influences on mental health and need to be further explored in systematic studies. We conjecture that such differences may arise from the evolutionary mechanisms that have shaped human societies to live and share in close physical connectedness. Such an effect has been previously shown in primates kept in isolation who display depressive symptoms [<xref ref-type="bibr" rid="ref23">23</xref>,<xref ref-type="bibr" rid="ref24">24</xref>]. Similarly, parenting and its association with neuropeptide hormones may partially explain [<xref ref-type="bibr" rid="ref25">25</xref>] our results that the presence of kids reduces anxiety levels. Interestingly, the COVID-19 pandemic has created a unique natural experiment on the collective mental health response of individuals to a health emergency.</p>
      <p>The life cycle of such a response may need to be further studied as the world goes through various phases of the pandemic until its resolution. However, our study indicates that the mental health impact is observable within a span of a few months, especially on young individuals. Further research will be needed, ideally in a longitudinal setting, where the same individuals can be surveyed again to understand the dynamics of the collective mental health response.</p>
      <p>Our results also highlight that modern technological development in virtual communication is not able to replace natural socializing. Hence, it becomes imperative to design better and more empathetic technological tools that may shape a society and prevent isolation and alienation even while maintaining physical distancing and preventive measures for limiting spread. Personalization and contextualization of such measures will also be important, as our results indicate that persons with previous mental health conditions may be disproportionately affected.</p>
      <p>Finally, our results indicate that it may be possible to identify people at the highest risk of developing mental health disturbances. Our model achieved its best performance for those who were most vulnerable (having mental stress more than 5 days in a week) versus least vulnerable (having no stress or less than 1 day of stress in a week). This can help in the segmentation of vulnerable populations such as frontline health care workers and those who are facing disproportionately higher levels of stress during this time.</p>
      <p>A key factor in clinical and public health models is transparency and explainability in the face of complex interactions. Mental health variables are expected to have complex dependencies with potential confounding factors, mediation, and intercausal dependency; therefore, we extended our association analysis with data-driven structure learning of a BN. We preferred this approach over black box machine learning and standard statistical modeling for several reasons. Structure learning allows us to discover and model confounding factors transparently, whereas black box machine learning models such as RFs and gradient boosted machines are not well suited for transparent reasoning. Standard statistical approaches make it humanly impossible to model interactions among hundreds of variables. Structure learning allows discovery and dissection of interactions into mediation, confounding, and intercausal effects. The challenge of incorrect learning is addressed by ensembling many BNs (101 in our case) and choosing the ensemble voted structure. Our artificial intelligence (AI) approach has earlier been validated for public health problems [<xref ref-type="bibr" rid="ref26">26</xref>,<xref ref-type="bibr" rid="ref27">27</xref>], and this study demonstrates the underexplored potential of such an approach in complex mental health scenarios.</p>
      <p>Our study has a few limitations. Establishing causal inference in cross-sectional data is nearly impossible, and we acknowledge the possibility of confounders. However, this was precisely the reason we chose the structure learning approach, as some of the confounding influences can be transparently discovered and explained. The ensemble voted structure over the sufficiently large number of bootstrapped structures is expected to be robust, as a set of 101 BNs was found to be sufficiently large enough for this study to address the challenge of incorrect learning. Our approach is best suited as a probabilistic reasoning model to explain mental health determinants and to make predictions, a useful outcome in COVID-19–induced mental health morbidity. We could not explain why anxiety levels may be lower in persons with pre-existing cancer or hypertension. This may be a result of reduced work environment–related stress or more contact with family members at home. However, the current data set is not suited to address this at a finer level of explainability. In addition, we could not comment upon the temporality and persistence of these effects. Our results are currently limited to only one geography (ie, the United States). However, the relatively large sample size and multiethnic involvement in the survey makes the model representative for most of the ethnicities and influences across the United States; hence, it is likely to hold true in the United States. Finally, we believe that our study contributes to the use of explainable AI to predict mental health at a population level using survey data, hence making it broadly applicable. Survey data sets are notoriously noisy, and our approach achieved a balance between knowledge discovery and a predictive accuracy of 80%, thus establishing a baseline under a novel scenario. Our algorithms can be used as a screening method for identifying individuals who need help, and further studies with additional measurements and features may increase the accuracy of predictions. Therefore, predictive models for screening and assessing the mental health impact of COVID-19 is a crucial step toward proactive management and prevention of psychiatric comorbidities as populations continue to fight the pandemic.</p>
    </sec>
  </body>
  <back>
    <app-group>
      <supplementary-material id="app1">
        <label>Multimedia Appendix 1</label>
        <p>Variable groups as indicators.</p>
        <media xlink:href="mental_v8i4e25097_app1.xlsx" xlink:title="XLSX File  (Microsoft Excel File), 11 KB"/>
      </supplementary-material>
      <supplementary-material id="app2">
        <label>Multimedia Appendix 2</label>
        <p>Supplementary figures.</p>
        <media xlink:href="mental_v8i4e25097_app2.pdf" xlink:title="PDF File  (Adobe PDF File), 239 KB"/>
      </supplementary-material>
    </app-group>
    <glossary>
      <title>Abbreviations</title>
      <def-list>
        <def-item>
          <term id="abb1">BN</term>
          <def>
            <p>Bayesian network</p>
          </def>
        </def-item>
        <def-item>
          <term id="abb2">RF</term>
          <def>
            <p>random forest</p>
          </def>
        </def-item>
        <def-item>
          <term id="abb3">SVM</term>
          <def>
            <p>support vector machine</p>
          </def>
        </def-item>
      </def-list>
    </glossary>
    <fn-group>
      <fn fn-type="con">
        <p>VK (vibhor@iiitd.ac.in) and TS (tavpriteshsethi@iiitd.ac.in) serve as co-corresponding authors of this article. VK, TS, and IPJ contributed to the study design. IPJ and AK contributed to the data set. IPJ and RA contributed to the data analysis. IPJ, RA, and TS contributed to the paper writing. VK and TS contributed to the paper review.</p>
      </fn>
      <fn fn-type="conflict">
        <p>None declared.</p>
      </fn>
    </fn-group>
    <ref-list>
      <ref id="ref1">
        <label>1</label>
        <nlm-citation citation-type="web">
          <article-title>COVID-19: stimulating the economy and employment: as jobs crisis deepens, ILO warns of uncertain and incomplete labour market recovery</article-title>
          <source>International Labour Organization</source>
          <access-date>2020-08-06</access-date>
          <comment>
            <ext-link ext-link-type="uri" xlink:type="simple" xlink:href="https://www.ilo.org/global/about-the-ilo/newsroom/news/WCMS_749398/lang--en/index.htm">https://www.ilo.org/global/about-the-ilo/newsroom/news/WCMS_749398/lang--en/index.htm</ext-link>
          </comment>
        </nlm-citation>
      </ref>
      <ref id="ref2">
        <label>2</label>
        <nlm-citation citation-type="journal">
          <person-group person-group-type="author">
            <name name-style="western">
              <surname>Mrklas</surname>
              <given-names>K</given-names>
            </name>
            <name name-style="western">
              <surname>Shalaby</surname>
              <given-names>R</given-names>
            </name>
            <name name-style="western">
              <surname>Hrabok</surname>
              <given-names>M</given-names>
            </name>
            <name name-style="western">
              <surname>Gusnowski</surname>
              <given-names>A</given-names>
            </name>
            <name name-style="western">
              <surname>Vuong</surname>
              <given-names>W</given-names>
            </name>
            <name name-style="western">
              <surname>Surood</surname>
              <given-names>S</given-names>
            </name>
            <name name-style="western">
              <surname>Urichuk</surname>
              <given-names>L</given-names>
            </name>
            <name name-style="western">
              <surname>Li</surname>
              <given-names>D</given-names>
            </name>
            <name name-style="western">
              <surname>Li</surname>
              <given-names>XM</given-names>
            </name>
            <name name-style="western">
              <surname>Greenshaw</surname>
              <given-names>AJ</given-names>
            </name>
            <name name-style="western">
              <surname>Agyapong</surname>
              <given-names>VIO</given-names>
            </name>
          </person-group>
          <article-title>Prevalence of perceived stress, anxiety, depression, and obsessive-compulsive symptoms in health care workers and other workers in Alberta during the COVID-19 pandemic: cross-sectional survey</article-title>
          <source>JMIR Ment Health</source>
          <year>2020</year>
          <month>09</month>
          <day>25</day>
          <volume>7</volume>
          <issue>9</issue>
          <fpage>e22408</fpage>
          <comment>
            <ext-link ext-link-type="uri" xlink:type="simple" xlink:href="https://mental.jmir.org/2020/9/e22408/"/>
          </comment>
          <pub-id pub-id-type="doi">10.2196/22408</pub-id>
          <pub-id pub-id-type="medline">32915764</pub-id>
          <pub-id pub-id-type="pii">v7i9e22408</pub-id>
          <pub-id pub-id-type="pmcid">PMC7527165</pub-id>
        </nlm-citation>
      </ref>
      <ref id="ref3">
        <label>3</label>
        <nlm-citation citation-type="journal">
          <person-group person-group-type="author">
            <collab>The Lancet Psychiatry</collab>
          </person-group>
          <article-title>Mental health and COVID-19: change the conversation</article-title>
          <source>Lancet Psychiatry</source>
          <year>2020</year>
          <month>06</month>
          <volume>7</volume>
          <issue>6</issue>
          <fpage>463</fpage>
          <comment>
            <ext-link ext-link-type="uri" xlink:type="simple" xlink:href="http://europepmc.org/abstract/MED/32380007"/>
          </comment>
          <pub-id pub-id-type="doi">10.1016/S2215-0366(20)30194-2</pub-id>
          <pub-id pub-id-type="medline">32380007</pub-id>
          <pub-id pub-id-type="pii">S2215-0366(20)30194-2</pub-id>
          <pub-id pub-id-type="pmcid">PMC7198177</pub-id>
        </nlm-citation>
      </ref>
      <ref id="ref4">
        <label>4</label>
        <nlm-citation citation-type="journal">
          <person-group person-group-type="author">
            <name name-style="western">
              <surname>Pereira-Sanchez</surname>
              <given-names>V</given-names>
            </name>
            <name name-style="western">
              <surname>Adiukwu</surname>
              <given-names>F</given-names>
            </name>
            <name name-style="western">
              <surname>El Hayek</surname>
              <given-names>S</given-names>
            </name>
            <name name-style="western">
              <surname>Bytyçi</surname>
              <given-names>DG</given-names>
            </name>
            <name name-style="western">
              <surname>Gonzalez-Diaz</surname>
              <given-names>JM</given-names>
            </name>
            <name name-style="western">
              <surname>Kundadak</surname>
              <given-names>GK</given-names>
            </name>
            <name name-style="western">
              <surname>Larnaout</surname>
              <given-names>A</given-names>
            </name>
            <name name-style="western">
              <surname>Nofal</surname>
              <given-names>M</given-names>
            </name>
            <name name-style="western">
              <surname>Orsolini</surname>
              <given-names>L</given-names>
            </name>
            <name name-style="western">
              <surname>Ramalho</surname>
              <given-names>R</given-names>
            </name>
            <name name-style="western">
              <surname>Ransing</surname>
              <given-names>R</given-names>
            </name>
            <name name-style="western">
              <surname>Shalbafan</surname>
              <given-names>M</given-names>
            </name>
            <name name-style="western">
              <surname>Soler-Vidal</surname>
              <given-names>J</given-names>
            </name>
            <name name-style="western">
              <surname>Syarif</surname>
              <given-names>Z</given-names>
            </name>
            <name name-style="western">
              <surname>Teixeira</surname>
              <given-names>ALS</given-names>
            </name>
            <name name-style="western">
              <surname>da Costa</surname>
              <given-names>MP</given-names>
            </name>
          </person-group>
          <article-title>COVID-19 effect on mental health: patients and workforce</article-title>
          <source>Lancet Psychiatry</source>
          <year>2020</year>
          <month>06</month>
          <volume>7</volume>
          <issue>6</issue>
          <fpage>e29</fpage>
          <lpage>e30</lpage>
          <comment>
            <ext-link ext-link-type="uri" xlink:type="simple" xlink:href="http://europepmc.org/abstract/MED/32445691"/>
          </comment>
          <pub-id pub-id-type="doi">10.1016/S2215-0366(20)30153-X</pub-id>
          <pub-id pub-id-type="medline">32445691</pub-id>
          <pub-id pub-id-type="pii">S2215-0366(20)30153-X</pub-id>
          <pub-id pub-id-type="pmcid">PMC7239628</pub-id>
        </nlm-citation>
      </ref>
      <ref id="ref5">
        <label>5</label>
        <nlm-citation citation-type="journal">
          <person-group person-group-type="author">
            <name name-style="western">
              <surname>Chen</surname>
              <given-names>Q</given-names>
            </name>
            <name name-style="western">
              <surname>Liang</surname>
              <given-names>M</given-names>
            </name>
            <name name-style="western">
              <surname>Li</surname>
              <given-names>Y</given-names>
            </name>
            <name name-style="western">
              <surname>Guo</surname>
              <given-names>J</given-names>
            </name>
            <name name-style="western">
              <surname>Fei</surname>
              <given-names>D</given-names>
            </name>
            <name name-style="western">
              <surname>Wang</surname>
              <given-names>L</given-names>
            </name>
            <name name-style="western">
              <surname>He</surname>
              <given-names>L</given-names>
            </name>
            <name name-style="western">
              <surname>Sheng</surname>
              <given-names>C</given-names>
            </name>
            <name name-style="western">
              <surname>Cai</surname>
              <given-names>Y</given-names>
            </name>
            <name name-style="western">
              <surname>Li</surname>
              <given-names>X</given-names>
            </name>
            <name name-style="western">
              <surname>Wang</surname>
              <given-names>J</given-names>
            </name>
            <name name-style="western">
              <surname>Zhang</surname>
              <given-names>Z</given-names>
            </name>
          </person-group>
          <article-title>Mental health care for medical staff in China during the COVID-19 outbreak</article-title>
          <source>Lancet Psychiatry</source>
          <year>2020</year>
          <month>04</month>
          <volume>7</volume>
          <issue>4</issue>
          <fpage>e15</fpage>
          <lpage>e16</lpage>
          <comment>
            <ext-link ext-link-type="uri" xlink:type="simple" xlink:href="http://europepmc.org/abstract/MED/32085839"/>
          </comment>
          <pub-id pub-id-type="doi">10.1016/S2215-0366(20)30078-X</pub-id>
          <pub-id pub-id-type="medline">32085839</pub-id>
          <pub-id pub-id-type="pii">S2215-0366(20)30078-X</pub-id>
          <pub-id pub-id-type="pmcid">PMC7129426</pub-id>
        </nlm-citation>
      </ref>
      <ref id="ref6">
        <label>6</label>
        <nlm-citation citation-type="journal">
          <person-group person-group-type="author">
            <name name-style="western">
              <surname>Mahase</surname>
              <given-names>E</given-names>
            </name>
          </person-group>
          <article-title>Covid-19: mental health consequences of pandemic need urgent research, paper advises</article-title>
          <source>BMJ</source>
          <year>2020</year>
          <month>04</month>
          <day>16</day>
          <volume>369</volume>
          <fpage>m1515</fpage>
          <pub-id pub-id-type="doi">10.1136/bmj.m1515</pub-id>
          <pub-id pub-id-type="medline">32299806</pub-id>
        </nlm-citation>
      </ref>
      <ref id="ref7">
        <label>7</label>
        <nlm-citation citation-type="journal">
          <person-group person-group-type="author">
            <name name-style="western">
              <surname>Pfefferbaum</surname>
              <given-names>B</given-names>
            </name>
            <name name-style="western">
              <surname>North</surname>
              <given-names>CS</given-names>
            </name>
          </person-group>
          <article-title>Mental health and the Covid-19 pandemic</article-title>
          <source>N Engl J Med</source>
          <year>2020</year>
          <month>08</month>
          <day>06</day>
          <volume>383</volume>
          <issue>6</issue>
          <fpage>510</fpage>
          <lpage>512</lpage>
          <pub-id pub-id-type="doi">10.1056/NEJMp2008017</pub-id>
          <pub-id pub-id-type="medline">32283003</pub-id>
        </nlm-citation>
      </ref>
      <ref id="ref8">
        <label>8</label>
        <nlm-citation citation-type="journal">
          <person-group person-group-type="author">
            <name name-style="western">
              <surname>Seabrook</surname>
              <given-names>EM</given-names>
            </name>
            <name name-style="western">
              <surname>Kern</surname>
              <given-names>ML</given-names>
            </name>
            <name name-style="western">
              <surname>Rickard</surname>
              <given-names>NS</given-names>
            </name>
          </person-group>
          <article-title>Social networking sites, depression, and anxiety: a systematic review</article-title>
          <source>JMIR Ment Health</source>
          <year>2016</year>
          <month>11</month>
          <day>23</day>
          <volume>3</volume>
          <issue>4</issue>
          <fpage>e50</fpage>
          <comment>
            <ext-link ext-link-type="uri" xlink:type="simple" xlink:href="https://mental.jmir.org/2016/4/e50/"/>
          </comment>
          <pub-id pub-id-type="doi">10.2196/mental.5842</pub-id>
          <pub-id pub-id-type="medline">27881357</pub-id>
          <pub-id pub-id-type="pii">v3i4e50</pub-id>
          <pub-id pub-id-type="pmcid">PMC5143470</pub-id>
        </nlm-citation>
      </ref>
      <ref id="ref9">
        <label>9</label>
        <nlm-citation citation-type="journal">
          <person-group person-group-type="author">
            <name name-style="western">
              <surname>Bruen</surname>
              <given-names>AJ</given-names>
            </name>
            <name name-style="western">
              <surname>Wall</surname>
              <given-names>A</given-names>
            </name>
            <name name-style="western">
              <surname>Haines-Delmont</surname>
              <given-names>A</given-names>
            </name>
            <name name-style="western">
              <surname>Perkins</surname>
              <given-names>E</given-names>
            </name>
          </person-group>
          <article-title>Exploring suicidal ideation using an innovative mobile app-strength within me: the usability and acceptability of setting up a trial involving mobile technology and mental health service users</article-title>
          <source>JMIR Ment Health</source>
          <year>2020</year>
          <month>09</month>
          <day>28</day>
          <volume>7</volume>
          <issue>9</issue>
          <fpage>e18407</fpage>
          <comment>
            <ext-link ext-link-type="uri" xlink:type="simple" xlink:href="https://mental.jmir.org/2020/9/e18407/"/>
          </comment>
          <pub-id pub-id-type="doi">10.2196/18407</pub-id>
          <pub-id pub-id-type="medline">32985995</pub-id>
          <pub-id pub-id-type="pii">v7i9e18407</pub-id>
          <pub-id pub-id-type="pmcid">PMC7551108</pub-id>
        </nlm-citation>
      </ref>
      <ref id="ref10">
        <label>10</label>
        <nlm-citation citation-type="journal">
          <person-group person-group-type="author">
            <name name-style="western">
              <surname>Gao</surname>
              <given-names>J</given-names>
            </name>
            <name name-style="western">
              <surname>Zheng</surname>
              <given-names>P</given-names>
            </name>
            <name name-style="western">
              <surname>Jia</surname>
              <given-names>Y</given-names>
            </name>
            <name name-style="western">
              <surname>Chen</surname>
              <given-names>H</given-names>
            </name>
            <name name-style="western">
              <surname>Mao</surname>
              <given-names>Y</given-names>
            </name>
            <name name-style="western">
              <surname>Chen</surname>
              <given-names>S</given-names>
            </name>
            <name name-style="western">
              <surname>Wang</surname>
              <given-names>Y</given-names>
            </name>
            <name name-style="western">
              <surname>Fu</surname>
              <given-names>H</given-names>
            </name>
            <name name-style="western">
              <surname>Dai</surname>
              <given-names>J</given-names>
            </name>
          </person-group>
          <article-title>Mental health problems and social media exposure during COVID-19 outbreak</article-title>
          <source>PLoS One</source>
          <year>2020</year>
          <volume>15</volume>
          <issue>4</issue>
          <fpage>e0231924</fpage>
          <comment>
            <ext-link ext-link-type="uri" xlink:type="simple" xlink:href="https://dx.plos.org/10.1371/journal.pone.0231924"/>
          </comment>
          <pub-id pub-id-type="doi">10.1371/journal.pone.0231924</pub-id>
          <pub-id pub-id-type="medline">32298385</pub-id>
          <pub-id pub-id-type="pii">PONE-D-20-06332</pub-id>
          <pub-id pub-id-type="pmcid">PMC7162477</pub-id>
        </nlm-citation>
      </ref>
      <ref id="ref11">
        <label>11</label>
        <nlm-citation citation-type="web">
          <source>COVID Impact Survey</source>
          <access-date>2020-05-25</access-date>
          <comment>
            <ext-link ext-link-type="uri" xlink:type="simple" xlink:href="https://www.covid-impact.org">https://www.covid-impact.org</ext-link>
          </comment>
        </nlm-citation>
      </ref>
      <ref id="ref12">
        <label>12</label>
        <nlm-citation citation-type="web">
          <article-title>About the survey questionnaire</article-title>
          <source>COVID Impact Survey</source>
          <access-date>2020-05-25</access-date>
          <comment>
            <ext-link ext-link-type="uri" xlink:type="simple" xlink:href="https://www.covid-impact.org/about-the-survey-questionnaire">https://www.covid-impact.org/about-the-survey-questionnaire</ext-link>
          </comment>
        </nlm-citation>
      </ref>
      <ref id="ref13">
        <label>13</label>
        <nlm-citation citation-type="web">
          <person-group person-group-type="author">
            <name name-style="western">
              <surname>Revelle</surname>
              <given-names>W</given-names>
            </name>
          </person-group>
          <article-title>psych: procedures for psychological, psychometric, and personality research</article-title>
          <source>The Comprehensive R Archive Network</source>
          <year>2020</year>
          <access-date>2020-06-26</access-date>
          <comment>
            <ext-link ext-link-type="uri" xlink:type="simple" xlink:href="https://CRAN.R-project.org/package=psych">https://CRAN.R-project.org/package=psych</ext-link>
          </comment>
        </nlm-citation>
      </ref>
      <ref id="ref14">
        <label>14</label>
        <nlm-citation citation-type="journal">
          <person-group person-group-type="author">
            <name name-style="western">
              <surname>Gámez</surname>
              <given-names>JA</given-names>
            </name>
            <name name-style="western">
              <surname>Mateo</surname>
              <given-names>JL</given-names>
            </name>
            <name name-style="western">
              <surname>Puerta</surname>
              <given-names>JM</given-names>
            </name>
          </person-group>
          <article-title>Learning Bayesian networks by hill climbing: efficient methods based on progressive restriction of the neighborhood</article-title>
          <source>Data Mining Knowledge Discovery</source>
          <year>2010</year>
          <month>5</month>
          <day>11</day>
          <volume>22</volume>
          <issue>1-2</issue>
          <fpage>106</fpage>
          <lpage>148</lpage>
          <pub-id pub-id-type="doi">10.1007/s10618-010-0178-6</pub-id>
        </nlm-citation>
      </ref>
      <ref id="ref15">
        <label>15</label>
        <nlm-citation citation-type="journal">
          <person-group person-group-type="author">
            <name name-style="western">
              <surname>Bozdogan</surname>
              <given-names>H</given-names>
            </name>
          </person-group>
          <article-title>Model selection and Akaike's Information Criterion (AIC): the general theory and its analytical extensions</article-title>
          <source>Psychometrika</source>
          <year>1987</year>
          <month>9</month>
          <volume>52</volume>
          <issue>3</issue>
          <fpage>345</fpage>
          <lpage>370</lpage>
          <pub-id pub-id-type="doi">10.1007/BF02294361</pub-id>
        </nlm-citation>
      </ref>
      <ref id="ref16">
        <label>16</label>
        <nlm-citation citation-type="web">
          <person-group person-group-type="author">
            <name name-style="western">
              <surname>Yedidia</surname>
              <given-names>JS</given-names>
            </name>
            <name name-style="western">
              <surname>Freeman</surname>
              <given-names>WT</given-names>
            </name>
            <name name-style="western">
              <surname>Weiss</surname>
              <given-names>Y</given-names>
            </name>
          </person-group>
          <article-title>Generalized belief propagation</article-title>
          <source>NeurIPS Proceedings</source>
          <year>2001</year>
          <access-date>2020-06-28</access-date>
          <comment>
            <ext-link ext-link-type="uri" xlink:type="simple" xlink:href="http://papers.nips.cc/paper/1832-generalized-belief-propagation.pdf">http://papers.nips.cc/paper/1832-generalized-belief-propagation.pdf</ext-link>
          </comment>
        </nlm-citation>
      </ref>
      <ref id="ref17">
        <label>17</label>
        <nlm-citation citation-type="web">
          <person-group person-group-type="author">
            <name name-style="western">
              <surname>Sethi</surname>
              <given-names>T</given-names>
            </name>
            <name name-style="western">
              <surname>Maheshwari</surname>
              <given-names>S</given-names>
            </name>
          </person-group>
          <article-title>wiseR: a shiny application for end-to-end Bayesian decision network analysis and web-deployment</article-title>
          <source>The Comprehensive R Archive Network</source>
          <year>2018</year>
          <access-date>2020-05-08</access-date>
          <comment>
            <ext-link ext-link-type="uri" xlink:type="simple" xlink:href="https://CRAN.R-project.org/package=wiseR">https://CRAN.R-project.org/package=wiseR</ext-link>
          </comment>
        </nlm-citation>
      </ref>
      <ref id="ref18">
        <label>18</label>
        <nlm-citation citation-type="book">
          <person-group person-group-type="author">
            <name name-style="western">
              <surname>Koller</surname>
              <given-names>D</given-names>
            </name>
            <name name-style="western">
              <surname>Friedman</surname>
              <given-names>N</given-names>
            </name>
          </person-group>
          <source>Probabilistic Graphical Models: Principles and Techniques</source>
          <year>2009</year>
          <publisher-loc>Cambridge, MA</publisher-loc>
          <publisher-name>MIT Press</publisher-name>
        </nlm-citation>
      </ref>
      <ref id="ref19">
        <label>19</label>
        <nlm-citation citation-type="journal">
          <person-group person-group-type="author">
            <name name-style="western">
              <surname>Chawla</surname>
              <given-names>NV</given-names>
            </name>
            <name name-style="western">
              <surname>Bowyer</surname>
              <given-names>KW</given-names>
            </name>
            <name name-style="western">
              <surname>Hall</surname>
              <given-names>LO</given-names>
            </name>
            <name name-style="western">
              <surname>Kegelmeyer</surname>
              <given-names>WP</given-names>
            </name>
          </person-group>
          <article-title>SMOTE: synthetic minority over-sampling technique</article-title>
          <source>J Artif Intelligence Res</source>
          <year>2002</year>
          <month>06</month>
          <day>01</day>
          <volume>16</volume>
          <fpage>321</fpage>
          <lpage>357</lpage>
          <pub-id pub-id-type="doi">10.1613/jair.953</pub-id>
        </nlm-citation>
      </ref>
      <ref id="ref20">
        <label>20</label>
        <nlm-citation citation-type="web">
          <source>scikit-learn</source>
          <access-date>2020-06-26</access-date>
          <comment>
            <ext-link ext-link-type="uri" xlink:type="simple" xlink:href="https://scikit-learn.org/stable/">https://scikit-learn.org/stable/</ext-link>
          </comment>
        </nlm-citation>
      </ref>
      <ref id="ref21">
        <label>21</label>
        <nlm-citation citation-type="journal">
          <person-group person-group-type="author">
            <name name-style="western">
              <surname>Bilsen</surname>
              <given-names>J</given-names>
            </name>
          </person-group>
          <article-title>Suicide and youth: risk factors</article-title>
          <source>Front Psychiatry</source>
          <year>2018</year>
          <volume>9</volume>
          <fpage>540</fpage>
          <pub-id pub-id-type="doi">10.3389/fpsyt.2018.00540</pub-id>
          <pub-id pub-id-type="medline">30425663</pub-id>
          <pub-id pub-id-type="pmcid">PMC6218408</pub-id>
        </nlm-citation>
      </ref>
      <ref id="ref22">
        <label>22</label>
        <nlm-citation citation-type="journal">
          <person-group person-group-type="author">
            <name name-style="western">
              <surname>Brådvik</surname>
              <given-names>L</given-names>
            </name>
          </person-group>
          <article-title>Suicide risk and mental disorders</article-title>
          <source>Int J Environ Res Public Health</source>
          <year>2018</year>
          <month>09</month>
          <day>17</day>
          <volume>15</volume>
          <issue>9</issue>
          <fpage>2028</fpage>
          <comment>
            <ext-link ext-link-type="uri" xlink:type="simple" xlink:href="https://www.mdpi.com/resolver?pii=ijerph15092028"/>
          </comment>
          <pub-id pub-id-type="doi">10.3390/ijerph15092028</pub-id>
          <pub-id pub-id-type="medline">30227658</pub-id>
          <pub-id pub-id-type="pii">ijerph15092028</pub-id>
          <pub-id pub-id-type="pmcid">PMC6165520</pub-id>
        </nlm-citation>
      </ref>
      <ref id="ref23">
        <label>23</label>
        <nlm-citation citation-type="journal">
          <person-group person-group-type="author">
            <name name-style="western">
              <surname>Hennessy</surname>
              <given-names>MB</given-names>
            </name>
            <name name-style="western">
              <surname>McCowan</surname>
              <given-names>B</given-names>
            </name>
            <name name-style="western">
              <surname>Jiang</surname>
              <given-names>J</given-names>
            </name>
            <name name-style="western">
              <surname>Capitanio</surname>
              <given-names>JP</given-names>
            </name>
          </person-group>
          <article-title>Depressive-like behavioral response of adult male rhesus monkeys during routine animal husbandry procedure</article-title>
          <source>Front Behav Neurosci</source>
          <year>2014</year>
          <volume>8</volume>
          <fpage>309</fpage>
          <pub-id pub-id-type="doi">10.3389/fnbeh.2014.00309</pub-id>
          <pub-id pub-id-type="medline">25249954</pub-id>
          <pub-id pub-id-type="pmcid">PMC4159029</pub-id>
        </nlm-citation>
      </ref>
      <ref id="ref24">
        <label>24</label>
        <nlm-citation citation-type="journal">
          <person-group person-group-type="author">
            <name name-style="western">
              <surname>McKinney</surname>
              <given-names>WT</given-names>
            </name>
          </person-group>
          <article-title>Primate social isolation: psychiatric implications</article-title>
          <source>Arch Gen Psychiatry</source>
          <year>1974</year>
          <month>09</month>
          <volume>31</volume>
          <issue>3</issue>
          <fpage>422</fpage>
          <lpage>6</lpage>
          <pub-id pub-id-type="doi">10.1001/archpsyc.1974.01760150122018</pub-id>
          <pub-id pub-id-type="medline">4415204</pub-id>
        </nlm-citation>
      </ref>
      <ref id="ref25">
        <label>25</label>
        <nlm-citation citation-type="journal">
          <person-group person-group-type="author">
            <name name-style="western">
              <surname>Kohl</surname>
              <given-names>J</given-names>
            </name>
            <name name-style="western">
              <surname>Autry</surname>
              <given-names>AE</given-names>
            </name>
            <name name-style="western">
              <surname>Dulac</surname>
              <given-names>C</given-names>
            </name>
          </person-group>
          <article-title>The neurobiology of parenting: a neural circuit perspective</article-title>
          <source>Bioessays</source>
          <year>2017</year>
          <month>01</month>
          <volume>39</volume>
          <issue>1</issue>
          <fpage>1</fpage>
          <lpage>11</lpage>
          <comment>
            <ext-link ext-link-type="uri" xlink:type="simple" xlink:href="http://europepmc.org/abstract/MED/27921311"/>
          </comment>
          <pub-id pub-id-type="doi">10.1002/bies.201600159</pub-id>
          <pub-id pub-id-type="medline">27921311</pub-id>
          <pub-id pub-id-type="pmcid">PMC5241891</pub-id>
        </nlm-citation>
      </ref>
      <ref id="ref26">
        <label>26</label>
        <nlm-citation citation-type="journal">
          <person-group person-group-type="author">
            <name name-style="western">
              <surname>Sethi</surname>
              <given-names>T</given-names>
            </name>
            <name name-style="western">
              <surname>Mittal</surname>
              <given-names>A</given-names>
            </name>
            <name name-style="western">
              <surname>Maheshwari</surname>
              <given-names>S</given-names>
            </name>
            <name name-style="western">
              <surname>Chugh</surname>
              <given-names>S</given-names>
            </name>
          </person-group>
          <article-title>Learning to address health inequality in the United States with a Bayesian decision network</article-title>
          <source>Proc AAAI Conference Artif Intelligence</source>
          <year>2019</year>
          <month>07</month>
          <day>17</day>
          <volume>33</volume>
          <fpage>710</fpage>
          <lpage>717</lpage>
          <pub-id pub-id-type="doi">10.1609/aaai.v33i01.3301710</pub-id>
        </nlm-citation>
      </ref>
      <ref id="ref27">
        <label>27</label>
        <nlm-citation citation-type="journal">
          <person-group person-group-type="author">
            <name name-style="western">
              <surname>Awasthi</surname>
              <given-names>R</given-names>
            </name>
            <name name-style="western">
              <surname>Patel</surname>
              <given-names>P</given-names>
            </name>
            <name name-style="western">
              <surname>Joshi</surname>
              <given-names>V</given-names>
            </name>
            <name name-style="western">
              <surname>Karkal</surname>
              <given-names>S</given-names>
            </name>
            <name name-style="western">
              <surname>Sethi</surname>
              <given-names>T</given-names>
            </name>
          </person-group>
          <article-title>Learning explainable interventions to mitigate HIV transmission in sex workers across five states in India</article-title>
          <source>arXiv.</source>
          <comment>Preprint posted online on November 30, 2020.
            <ext-link ext-link-type="uri" xlink:type="simple" xlink:href="http://arxiv.org/abs/2012.01930"/>
          </comment>
        </nlm-citation>
      </ref>
    </ref-list>
  </back>
</article>
