<?xml version="1.0" encoding="utf-8"?>
<!DOCTYPE article PUBLIC "-//NLM//DTD Journal Publishing DTD v2.0 20040830//EN" "http://dtd.nlm.nih.gov/publishing/2.0/journalpublishing.dtd">
<article xmlns:xlink="http://www.w3.org/1999/xlink" article-type="research-article" dtd-version="2.0">
  <front>
    <journal-meta>
      <journal-id journal-id-type="publisher-id">JMH</journal-id>
      <journal-id journal-id-type="nlm-ta">JMIR Ment Health</journal-id>
      <journal-title>JMIR Mental Health</journal-title>
      <issn pub-type="epub">2368-7959</issn>
      <publisher>
        <publisher-name>JMIR Publications</publisher-name>
        <publisher-loc>Toronto, Canada</publisher-loc>
      </publisher>
    </journal-meta>
    <article-meta>
      <article-id pub-id-type="publisher-id">v6i9e14719</article-id>
      <article-id pub-id-type="pmid">31538938</article-id>
      <article-id pub-id-type="doi">10.2196/14719</article-id>
      <article-categories>
        <subj-group subj-group-type="heading">
          <subject>Original Paper</subject>
        </subj-group>
        <subj-group subj-group-type="article-type">
          <subject>Original Paper</subject>
        </subj-group>
      </article-categories>
      <title-group>
        <article-title>Technology-Enabled Person-Centered Mental Health Services Reform: Strategy for Implementation Science</article-title>
      </title-group>
      <contrib-group>
        <contrib contrib-type="editor">
          <name>
            <surname>Torous</surname>
            <given-names>John</given-names>
          </name>
        </contrib>
      </contrib-group>
      <contrib-group>
        <contrib contrib-type="reviewer">
          <name>
            <surname>Bradford</surname>
            <given-names>Dana</given-names>
          </name>
        </contrib>
        <contrib contrib-type="reviewer">
          <name>
            <surname>Urwyler</surname>
            <given-names>Prabitha</given-names>
          </name>
        </contrib>
      </contrib-group>
      <contrib-group>
        <contrib id="contrib1" contrib-type="author" corresp="yes">
          <name name-style="western">
            <surname>LaMonica</surname>
            <given-names>Haley M</given-names>
          </name>
          <degrees>BS, MA, PhD, ABPP-CN</degrees>
          <xref rid="aff1" ref-type="aff">1</xref>
          <address>
            <institution>Brain and Mind Centre</institution>
            <institution>The University of Sydney</institution>
            <addr-line>94 Mallett Street</addr-line>
            <addr-line>Camperdown, 2050</addr-line>
            <country>Australia</country>
            <phone>61 286276939</phone>
            <email>haley.lamonica@sydney.edu.au</email>
          </address>
          <ext-link ext-link-type="orcid">http://orcid.org/0000-0002-6563-5467</ext-link>
        </contrib>
        <contrib id="contrib2" contrib-type="author">
          <name name-style="western">
            <surname>Davenport</surname>
            <given-names>Tracey A</given-names>
          </name>
          <degrees>BA (Hons), eMBA</degrees>
          <xref rid="aff1" ref-type="aff">1</xref>
          <ext-link ext-link-type="orcid">http://orcid.org/0000-0003-4218-9238</ext-link>
        </contrib>
        <contrib id="contrib3" contrib-type="author">
          <name name-style="western">
            <surname>Braunstein</surname>
            <given-names>Katharine</given-names>
          </name>
          <degrees>BPsy (Hons)</degrees>
          <xref rid="aff2" ref-type="aff">2</xref>
          <ext-link ext-link-type="orcid">http://orcid.org/0000-0003-1032-9080</ext-link>
        </contrib>
        <contrib id="contrib4" contrib-type="author">
          <name name-style="western">
            <surname>Ottavio</surname>
            <given-names>Antonia</given-names>
          </name>
          <degrees>DipHthSc, BN</degrees>
          <xref rid="aff2" ref-type="aff">2</xref>
          <ext-link ext-link-type="orcid">http://orcid.org/0000-0001-8307-6200</ext-link>
        </contrib>
        <contrib id="contrib5" contrib-type="author">
          <name name-style="western">
            <surname>Piper</surname>
            <given-names>Sarah</given-names>
          </name>
          <degrees>GradDip, Psych (Hons)</degrees>
          <xref rid="aff1" ref-type="aff">1</xref>
          <ext-link ext-link-type="orcid">http://orcid.org/0000-0002-3344-0382</ext-link>
        </contrib>
        <contrib id="contrib6" contrib-type="author">
          <name name-style="western">
            <surname>Martin</surname>
            <given-names>Craig</given-names>
          </name>
          <degrees>BN, BASc, Grad Dip MH, MN (Clinical Leadership)</degrees>
          <xref rid="aff2" ref-type="aff">2</xref>
          <ext-link ext-link-type="orcid">http://orcid.org/0000-0002-3151-2872</ext-link>
        </contrib>
        <contrib id="contrib7" contrib-type="author">
          <name name-style="western">
            <surname>Hickie</surname>
            <given-names>Ian B</given-names>
          </name>
          <degrees>AM, MD, FRANZCP, FASSA</degrees>
          <xref rid="aff1" ref-type="aff">1</xref>
          <ext-link ext-link-type="orcid">http://orcid.org/0000-0001-8832-9895</ext-link>
        </contrib>
        <contrib id="contrib8" contrib-type="author">
          <name name-style="western">
            <surname>Cross</surname>
            <given-names>Shane</given-names>
          </name>
          <degrees>BPsy (Hons), MPsych, PhD</degrees>
          <xref rid="aff1" ref-type="aff">1</xref>
          <ext-link ext-link-type="orcid">http://orcid.org/0000-0002-5413-8342</ext-link>
        </contrib>
      </contrib-group>
      <aff id="aff1">
        <label>1</label>
        <institution>Brain and Mind Centre</institution>
        <institution>The University of Sydney</institution>
        <addr-line>Camperdown</addr-line>
        <country>Australia</country>
      </aff>
      <aff id="aff2">
        <label>2</label>
        <institution>InnoWell Pty Ltd</institution>
        <addr-line>Camperdown</addr-line>
        <country>Australia</country>
      </aff>
      <author-notes>
        <corresp>Corresponding Author: Haley M LaMonica <email>haley.lamonica@sydney.edu.au</email></corresp>
      </author-notes>
      <pub-date pub-type="collection">
        <month>09</month>
        <year>2019</year>
      </pub-date>
      <pub-date pub-type="epub">
        <day>19</day>
        <month>09</month>
        <year>2019</year>
      </pub-date>
      <volume>6</volume>
      <issue>9</issue>
      <elocation-id>e14719</elocation-id>
      <history>
        <date date-type="received">
          <day>17</day>
          <month>5</month>
          <year>2019</year>
        </date>
        <date date-type="rev-request">
          <day>8</day>
          <month>6</month>
          <year>2019</year>
        </date>
        <date date-type="rev-recd">
          <day>2</day>
          <month>7</month>
          <year>2019</year>
        </date>
        <date date-type="accepted">
          <day>23</day>
          <month>7</month>
          <year>2019</year>
        </date>
      </history>
      <copyright-statement>©Haley M LaMonica, Tracey A Davenport, Katharine Braunstein, Antonia Ottavio, Sarah Piper, Craig Martin, Ian B Hickie, Shane Cross. Originally published in JMIR Mental Health (http://mental.jmir.org), 19.09.2019. </copyright-statement>
      <copyright-year>2019</copyright-year>
      <license license-type="open-access" xlink:href="https://creativecommons.org/licenses/by/4.0/">
        <p>This is an open-access article distributed under the terms of the Creative Commons Attribution License (https://creativecommons.org/licenses/by/4.0/), which permits unrestricted use, distribution, and reproduction in any medium, provided the original work, first published in JMIR Mental Health, is properly cited. The complete bibliographic information, a link to the original publication on http://mental.jmir.org/, as well as this copyright and license information must be included.</p>
      </license>
      <self-uri xlink:href="https://mental.jmir.org/2019/9/e14719" xlink:type="simple"/>
      <abstract>
        <sec sec-type="background">
          <title>Background</title>
          <p>Health information technologies are being rapidly developed to improve the delivery of mental health care; however, a range of facilitators, barriers, and contextual conditions can impact the adoption and sustainment of these solutions. An implementation science protocol supports researchers to achieve primary effectiveness goals in relation to mental health services reform and aids in the optimization of implementation processes to promote quality health care, prolonging sustainability.</p>
        </sec>
        <sec sec-type="objective">
          <title>Objective</title>
          <p>The aim of this paper is to describe our implementation science protocol, which serves as a foundation by which to systematically guide the implementation of technology-enabled solutions in traditional face-to-face and Web-based mental health services, allowing for revisions over time on the basis of retrospective review and constructive feedback from the services in which the technology-enabled solutions are implemented.</p>
        </sec>
        <sec sec-type="methods">
          <title>Methods</title>
          <p>Our implementation science protocol comprises four phases. The primary objective of the scoping and feasibility phase (Phase 1) is to determine the alignment between the service partner and the quality improvement goals supported by the technology-enabled solution. This is followed by Phase 2, the local co-design and preimplementation phase, which aims to utilize co-design methodologies, including service pathway modelling, participatory design, and user (acceptance) testing, to determine how the solutions could be used to enhance the service. In Phase 3, implementation, the accepted solution is embedded in the mental health service to achieve better outcomes for consumers and their families as well as health professionals and service managers. Using iterative evaluative processes throughout Phase 3, the solution is continuously developed, designed, and refined during implementation to adapt to the changing needs of the stakeholders, including consumers with lived experience and their families as well as the service. Thus, the primary outcome of Phase 3 is the optimized technology-enabled solution that can be maintained in a service during the sustainment and scalability phase (Phase 4) for the purposes of mental health services reform.</p>
        </sec>
        <sec sec-type="results">
          <title>Results</title>
          <p>Funding for the protocol was provided by the Australian Government Department of Health in June of 2017 for a period of 3 years. At the time of this publication, the protocol had been initiated in 11 services, serving three populations, all of which are currently operating in Phase 3. The first results are expected to be submitted for publication in 2020.</p>
        </sec>
        <sec sec-type="conclusions">
          <title>Conclusions</title>
          <p>With the aim of improving mental health service quality, our implementation science protocol aids in the identification of factors that predict the likelihood of implementation success, as well as the development of strategies to proactively mitigate potential barriers to achieve better implementation outcomes. Putting in place a theoretically sound implementation science protocol is essential to facilitate the uptake of novel technology-enabled solutions and evidence-based practices into routine clinical practice for the purposes of improved outcomes.</p>
        </sec>
      </abstract>
      <kwd-group>
        <kwd>implementation science</kwd>
        <kwd>mental health</kwd>
        <kwd>health care reform</kwd>
        <kwd>technology</kwd>
        <kwd>community-based participatory research</kwd>
      </kwd-group>
    </article-meta>
  </front>
  <body>
    <sec sec-type="introduction">
      <title>Introduction</title>
      <sec>
        <title>Health Information Technologies and Mental Health Services Reform</title>
        <p>Globally, the mental health system is plagued by fundamental shortcomings, including delays in service provision, limitations in access, fragmented services, failure to utilize routine outcome monitoring, and provision of care that does not match the consumer’s level of need. New health information technologies (HITs) are being rapidly developed to improve the delivery of mental health care for both health professionals and consumers, as well as to better support self-management of care. For example, cognitive-behavioral therapy approaches have been incorporated into several apps and websites, such as MoodMission [<xref ref-type="bibr" rid="ref1">1</xref>] and CBT-i Coach [<xref ref-type="bibr" rid="ref2">2</xref>], to help consumers better self-manage their health and well-being, provide psychoeducation about areas of concern or difficulty, and enhance traditional face-to-face care. Unfortunately, it is all too common for both traditional clinical and more novel electronic or Web-based interventions found to be efficacious in research studies not to be associated with meaningful outcomes for consumers in clinical settings [<xref ref-type="bibr" rid="ref3">3</xref>]. This may partly relate to fundamental differences in the factors frequently associated with successful clinical trials (eg, highly standardized, homogenous participant sample, and control of possible confounding factors), relative to those that facilitate effectiveness in clinical practice or community settings (eg, flexibility in the intervention for providers and consumers, appropriateness for a broad consumer group across multiple clinical settings, and applicability for multiple conditions). However, it is also important to consider possible failures in the implementation of evidence-based interventions in practice. For example, a recent systematic review highlighted the failings of HIT to support the management of heart failure because of the complexities of providing care to consumers who are often older, with multiple comorbidities, more vulnerable with less support, and less technically savvy [<xref ref-type="bibr" rid="ref4">4</xref>]. Furthermore, Bont and Bal describe an HIT as being set up for success by the clinical sponsors but failing because of the impact on traditional notions of what it meant to be a good health professional [<xref ref-type="bibr" rid="ref5">5</xref>].</p>
      </sec>
      <sec>
        <title>Implementation of Health Information Technologies: Barriers and Facilitators</title>
        <p>As it relates to health care, implementation science is defined as the scientific study of methods to promote the systematic uptake of research and development (R&amp;D) findings and evidence-based practices into routine clinical practice, with the aim of improving the quality and effectiveness of health services and the care provided [<xref ref-type="bibr" rid="ref6">6</xref>]. There is increasing appreciation among leading research organizations regarding the need to develop and utilize effective methods of implementing efficacious or effective interventions for the purposes of improving health care quality and efficiency [<xref ref-type="bibr" rid="ref7">7</xref>-<xref ref-type="bibr" rid="ref9">9</xref>]. To facilitate successful implementation, our implementation science protocol serves as a strategic, high-level, long-range plan with 3 primary aims: (1) describe the implementation process, (2) explain the facilitators and barriers to implementation outcomes, and (3) evaluate implementation [<xref ref-type="bibr" rid="ref10">10</xref>].</p>
        <sec>
          <title>Potential Facilitators and Barriers to Successful Implementation</title>
          <p>The results of a recent systematic review highlighted the benefits of HITs on the quality and efficacy of health care, partly by facilitating adherence to guidelines or protocol-based care with the aid of embedded electronic decision support functions [<xref ref-type="bibr" rid="ref11">11</xref>]. Specific examples from the mental health field include consistent support for phone calls or short message service text messages about medications, resulting both in improved adherence and reductions in hospitalizations, as well as for the use of Web-based self-management tools to improve quality of life, mood, and social functioning [<xref ref-type="bibr" rid="ref12">12</xref>]. Handheld devices to support health care consultations have also been shown to improve satisfaction with care [<xref ref-type="bibr" rid="ref12">12</xref>]. Despite the unique and additive benefits of HITs to mental health service quality, previous studies have shown that a range of facilitators, barriers, and contextual conditions impact on implementation adoption and sustainment [<xref ref-type="bibr" rid="ref13">13</xref>-<xref ref-type="bibr" rid="ref16">16</xref>], potentially resulting in underutilization. Specifically, several recent reviews have documented potential barriers and facilitators to implementation processes [<xref ref-type="bibr" rid="ref17">17</xref>-<xref ref-type="bibr" rid="ref21">21</xref>], which can be divided into service-level, health professional, and individual factors. Notably, there is often a reciprocal relationship between facilitators and barriers, such that a facilitator that is not championed can quickly become a barrier and vice versa.</p>
        </sec>
        <sec>
          <title>Service Factors</title>
          <p>The importance of leadership from the senior service management, as well as at the local service level, is consistently highlighted as a potential facilitator to successful implementation [<xref ref-type="bibr" rid="ref17">17</xref>,<xref ref-type="bibr" rid="ref18">18</xref>,<xref ref-type="bibr" rid="ref21">21</xref>,<xref ref-type="bibr" rid="ref22">22</xref>]. Indeed, our experience implementing a prototype technology-enabled solution [<xref ref-type="bibr" rid="ref23">23</xref>], described in detail below, across 5 youth mental health services highlighted the benefit of strong local leadership to promote service innovation, as well as R&amp;D within a service [<xref ref-type="bibr" rid="ref24">24</xref>]. Senior leaders help ensure alignment between the technology and the service mission and help foster an organizational culture that is open to change in the service model and receptive to the technology-enabled solution [<xref ref-type="bibr" rid="ref17">17</xref>]. In this regard, involvement of local champions (ie, prominent and well-respected individuals in a service) in the co-design and co-development of the solution and its implementation is frequently emphasized as a facilitator of successful implementation [<xref ref-type="bibr" rid="ref18">18</xref>,<xref ref-type="bibr" rid="ref25">25</xref>]. In addition, executive sponsorship is essential to emphasize the organizational commitment and demonstrate support for service change and redevelopment, especially given that the underestimation of change management can also act as a barrier to successful implementation [<xref ref-type="bibr" rid="ref17">17</xref>,<xref ref-type="bibr" rid="ref20">20</xref>]. Service leaders engender a positive attitude toward change and encourage adoption of the technology-enabled solution by frontline staff and consumers. Contingencies need to be put in place to ensure that the local champion roles are always filled, regardless of staff turnover.</p>
          <p>Consideration should also be given to the degree of alignment between conventional service models and workflows and the solution, as misalignment represents a barrier to successful implementation, with higher degrees of misalignment increasing the complexity of the implementation [<xref ref-type="bibr" rid="ref26">26</xref>]. Furthermore, in a systematic review of electronic health implementations, workflow disruptions were found to be the most cited factor in determining the success or failure of an implementation [<xref ref-type="bibr" rid="ref20">20</xref>]. Collaboration and R&amp;D between researchers and service staff (eg, health professionals, administrators, and service managers), as well as consumers with lived experience and their families, in relation to the iterative co-design and co-development of the technology-enabled solution (including service model), are critically important components to implementation success and sustainability for mental health services reform [<xref ref-type="bibr" rid="ref15">15</xref>,<xref ref-type="bibr" rid="ref18">18</xref>,<xref ref-type="bibr" rid="ref20">20</xref>,<xref ref-type="bibr" rid="ref27">27</xref>,<xref ref-type="bibr" rid="ref28">28</xref>].</p>
          <p>Resource limitations, including lack of appropriate personnel and equipment, are routinely reported as barriers to implementation [<xref ref-type="bibr" rid="ref17">17</xref>,<xref ref-type="bibr" rid="ref21">21</xref>]. Interestingly, however, the availability of adequate resources within a service has not been found to be a specific facilitator of implementation. As summarized by Vis et al [<xref ref-type="bibr" rid="ref21">21</xref>], reliable access to information and communications technology (ICT), as well as any required interoperability with other existing technology within the service, such as an electronic medical record or client management system, is a key determinant of implementation.</p>
        </sec>
        <sec>
          <title>Health Professional Factors</title>
          <p>A number of factors related to health professionals have been identified as facilitators for successful implementation. For example, the co-design and configuration of the solution to fit the needs of health professionals foster buy-in and acceptance [<xref ref-type="bibr" rid="ref18">18</xref>,<xref ref-type="bibr" rid="ref21">21</xref>,<xref ref-type="bibr" rid="ref22">22</xref>]. Successful implementation is also facilitated by effective education and training of health professionals, which nurtures self-efficacy and capacity in the context of continuous on-the-ground support [<xref ref-type="bibr" rid="ref17">17</xref>,<xref ref-type="bibr" rid="ref18">18</xref>,<xref ref-type="bibr" rid="ref21">21</xref>,<xref ref-type="bibr" rid="ref22">22</xref>]. Indeed, research indicates that learning outcomes are enhanced when initial education and training are supplemented with ongoing implementation support [<xref ref-type="bibr" rid="ref29">29</xref>]. Several potential barriers to implementation at the level of health professionals have also been consistently reported in the literature, including negative staff attitudes, resistance to change, and changes to work practices [<xref ref-type="bibr" rid="ref17">17</xref>,<xref ref-type="bibr" rid="ref20">20</xref>-<xref ref-type="bibr" rid="ref22">22</xref>].</p>
        </sec>
        <sec>
          <title>Individual Factors</title>
          <p>Implementation theories consistently state the importance of considering consumers’ needs when designing any implementation processes intended to improve outcomes [<xref ref-type="bibr" rid="ref30">30</xref>,<xref ref-type="bibr" rid="ref31">31</xref>]. As such, the involvement of consumers with lived experience and their families in the co-design process is key to implementation success [<xref ref-type="bibr" rid="ref20">20</xref>]. Consideration of consumer preferences for and disparities in the use of digital devices and modes of technology (eg, email, app, and website) to connect with health services is essential [<xref ref-type="bibr" rid="ref32">32</xref>] when iteratively designing and developing technology-enabled solutions. Other identified facilitators include consideration of the convenience and appropriateness of the solution in addressing consumers’ health care needs [<xref ref-type="bibr" rid="ref21">21</xref>], the user friendliness of the solution [<xref ref-type="bibr" rid="ref22">22</xref>,<xref ref-type="bibr" rid="ref33">33</xref>], and the appropriate adaptations (ie, configuration) of the solution to fit the specific cultural needs of the populations seeking care [<xref ref-type="bibr" rid="ref18">18</xref>]. As drivers of acceptance [<xref ref-type="bibr" rid="ref21">21</xref>,<xref ref-type="bibr" rid="ref22">22</xref>] and empowerment for consumers with lived experience [<xref ref-type="bibr" rid="ref20">20</xref>], the combination of these factors is likely to promote successful implementation. Conversely, individual resistance or nonparticipation, recruitment and retention issues, individual concerns about privacy, confidentiality, and information security, and a failure to adapt solutions over time to meet individual needs [<xref ref-type="bibr" rid="ref17">17</xref>,<xref ref-type="bibr" rid="ref20">20</xref>,<xref ref-type="bibr" rid="ref22">22</xref>] are consistent barriers to successful implementation and, in turn, mental health services reform.</p>
        </sec>
      </sec>
      <sec>
        <title>Co-Design of the Technology-Enabled Solution</title>
        <p>As described in detail in a study by Davenport et al [<xref ref-type="bibr" rid="ref23">23</xref>], the co-designed InnoWell Platform was developed through Project Synergy (by InnoWell Pty Ltd) to collect, store, and report clinical data back to a consumer and their health professional to promote person-centered care, self-management, early intervention, shared decision making, and routine outcome monitoring (see <xref ref-type="boxed-text" rid="box1">Textbox 1</xref>).</p>
        <boxed-text id="box1" position="float">
          <title>Description of the InnoWell Platform as it is listed on the Australian Register of Therapeutic Goods (software as a medical device, class 1, ARTG ID 315030).</title>
          <p>The InnoWell Platform is a customizable digital tool that assists assessment, monitoring and management of mental health issues, and maintenance of wellbeing. It does this by collecting personal and health information from consumers and their service providers. This information is stored, scored, and reported back to consumers and their health professionals to promote collaborative care. The clinical content is determined in collaboration with the service provider who invited the consumer to use the platform. Importantly, the platform does not provide stand-alone medical or health advice, diagnosis, or treatment. Instead, it guides and supports (but does not direct) consumers and their health professionals to decide what may be suitable care options. Importantly, all care aligns with the existing clinical governance (eg, policies and procedures) of the service provider.</p>
        </boxed-text>
      </sec>
    </sec>
    <sec sec-type="methods">
      <title>Methods</title>
      <sec>
        <title>An Implementation Science Protocol for Local Mental Health Services Reform</title>
        <p>For the purposes of the technology-enabled solution, quality implementation is achieved when the deployment of the technology and service model into health care services meets the requirements and standards to achieve the desired outcome [<xref ref-type="bibr" rid="ref34">34</xref>], namely technology-enabled mental health services reform. With this aim in mind, our implementation science protocol incorporates elements from 3 sources, namely the Quality Implementation Framework [<xref ref-type="bibr" rid="ref29">29</xref>] and the Accelerated Creation-to-Sustainment model [<xref ref-type="bibr" rid="ref15">15</xref>], as well as learnings from our experience co-designing and implementing a prototype technology-enabled solution into primary mental health care settings with young people [<xref ref-type="bibr" rid="ref24">24</xref>,<xref ref-type="bibr" rid="ref35">35</xref>]. The primary objective is to design a standardized yet flexible implementation science protocol to serve as a foundation by which to systematically guide implementation efforts, allowing for revisions over time on the basis of retrospective review and constructive feedback from services in which the technology-enabled solution is to be implemented and the consumers who will be engaging with the solution. Our implementation science protocol comprises 4 phases. The objectives and outcomes of each phase are described in detail below.</p>
        <sec>
          <title>Phase 1: Scoping and Feasibility</title>
          <p>The primary objective of Phase 1 is to determine the fit between the aims of the potential mental health service partner and the quality improvement goals supported by the technology-enabled solution. To begin with, it is essential to ensure that the service leadership is invested in quality improvement at the individual level and to demonstrate the relationship between this aim and the components of the technology-enabled solution. Our protocol emphasizes the importance of engaging organizational leaders from the earliest steps of the process. Furthermore, it is crucial to understand the basic attributes of the service, including the following: the characteristics of the consumers who access the service for care, the qualifications and occupancy of the service health professionals, administrators and service managers, and the current hardware and ICT infrastructure (eg, electronic medical record, client management system, and availability of Wi-Fi). It is also important to identify those persons who will be responsible for facilitating the implementation process at the service, including making decisions related to configuration and customization of the technology-enabled solution and overseeing change management processes (eg, changes to the clinical pathway, user journey, and workflows) on the ground. Provided the key tasks outlined in the scoping and feasibility phase are addressed, transition to Phase 2 then occurs. Alternatively, the service may choose to address the internal service gaps identified in this phase before proceeding to the next phase.</p>
        </sec>
        <sec>
          <title>Phase 2: Co-Design and Preimplementation</title>
          <p>A key feature of the technology is that the functionality and content are configurable, which allows it to adapt easily to local contexts, as well as specialist clinical and population groups, and it thus meets the person-centered care needs of a wider range of people presenting to diverse mental health providers. The primary objective of this phase is to utilize novel and innovative co-design methodologies to develop the technology-enabled solution and service model, as well as to user (acceptance) test these solutions for mental health services reform.</p>
          <sec>
            <title>Participatory Design of the Technology Solution</title>
            <p>As previously described in detail by our team [<xref ref-type="bibr" rid="ref35">35</xref>-<xref ref-type="bibr" rid="ref37">37</xref>], the potential end users (eg, general population, consumers with lived experience and their families, health professionals, administrators, and service managers) help inform the development of the technology through the continuous and iterative use of participatory design (or co-design), knowledge translation, user (acceptance) testing, and rapid prototyping methodologies. These user-centered methodologies emphasize the involvement of individuals with lived experienced (eg, consumers, family members, and peer support workers) in the co-design process of the technology and the implementation process. As outlined by Mohr et al [<xref ref-type="bibr" rid="ref15">15</xref>], interactive and iterative participatory design methodologies with key stakeholders (eg, information gathering, clarification of user requirements, workflow observations, co-design workshops, and user testing) help adapt the technology to local needs. Furthermore, as outlined in <xref rid="figure1" ref-type="fig">Figure 1</xref>, our protocol incorporates iterative evaluation methods to promote the continuous development and design of the technology, as well as the implementation process for the service and consumers for whom they provide care. The process of refining the technology can occur iteratively through the Implementation and Sustainment phases [<xref ref-type="bibr" rid="ref15">15</xref>].</p>
            <fig id="figure1" position="float">
              <label>Figure 1</label>
              <caption>
                <p>Flow diagram of evaluative methods and processes to optimize technology-enabled solutions for mental health services reform.</p>
              </caption>
              <graphic xlink:href="mental_v6i9e14719_fig1.png" alt-version="no" mimetype="image" position="float" xlink:type="simple"/>
            </fig>
          </sec>
          <sec>
            <title>Co-Designing Technology-Enabled Service Models</title>
            <p>In parallel, a process of co-designing service models occurs with representative stakeholders. The preexisting service model and related staff roles and responsibilities are mapped by using a process of service modelling developed through R&amp;D. As illustrated in the hypothetical example shown in <xref rid="figure2" ref-type="fig">Figure 2</xref>, each aspect of an individual’s journey through the service is delineated, starting from the perspective of the individual seeking help and expanding when other service stakeholders interact with that individual through the care journey. Following the presentation of an individual’s experience of a prototypic version of the technology-enabled solution, the gaps between the current model offered by the service and the key features of high-quality mental health care are explored. A hypothesized technology-enhanced service model is then co-designed, intertwining established processes with the additional technology elements and processes (eg, Web-based intake assessment, video visit). The changes to workflows and practices for each of the affected stakeholders (eg, health professional, administrator, and service manager) are also noted throughout the model. Finally, existing service metrics used to report on service quality are reviewed (as available) across pre-established quality domains (eg, safety, efficiency, and accessibility); thereafter, agreement is reached on the metrics used to monitor service quality using the technology solution [<xref ref-type="bibr" rid="ref38">38</xref>].</p>
            <fig id="figure2" position="float">
              <label>Figure 2</label>
              <caption>
                <p>Hypothetical service models showing pre and post implementation of the technology-enabled solution.</p>
              </caption>
              <graphic xlink:href="mental_v6i9e14719_fig2.png" alt-version="no" mimetype="image" position="float" xlink:type="simple"/>
            </fig>
          </sec>
          <sec>
            <title>Education and Training</title>
            <p>The education and training needs of service staff across all roles (eg, health professionals, administrators, and service managers), as well as consumers, are scoped during Phase 2. First, education and training are provided in relation to the evidence-based digital, clinical, service, and safety elements essential to improving service quality. Second, training in the use of the technology-enabled solution includes all necessary information to use the technology effectively for all staff members, including both a comprehensive overview of the functionality and the components relevant to the roles in the service. As outlined in <xref ref-type="table" rid="table1">Table 1</xref>, education and training are provided before implementation, with ongoing education, training, and technical assistance available throughout Phase 3.</p>
          </sec>
          <sec>
            <title>Discovering Facilitators and Barriers Before Implementation</title>
            <p>In Phase 2, ongoing interactions with the team of stakeholders involved in the implementation and change management process, present opportunities for the active and passive identification of barriers to, and facilitators of, successful implementation not previously uncovered. <xref ref-type="table" rid="table1">Table 1</xref> highlights some of the potential barriers identified in previous studies and the mitigation strategies for each.</p>
            <table-wrap position="float" id="table1">
              <label>Table 1</label>
              <caption>
                <p>Potential service-specific barriers and mitigation strategies to implementation of our technology-enabled solutions.</p>
              </caption>
              <table width="1000" cellpadding="7" cellspacing="0" border="1" rules="groups" frame="hsides">
                <col width="500"/>
                <col width="500"/>
                <thead>
                  <tr valign="top">
                    <td>Barriers</td>
                    <td>Mitigation strategies</td>
                  </tr>
                </thead>
                <tbody>
                  <tr valign="top">
                    <td>Negative staff attitudes [<xref ref-type="bibr" rid="ref20">20</xref>,<xref ref-type="bibr" rid="ref21">21</xref>]; staff resistance to change [<xref ref-type="bibr" rid="ref17">17</xref>,<xref ref-type="bibr" rid="ref20">20</xref>-<xref ref-type="bibr" rid="ref22">22</xref>]; changes to work practices, such as increased workload [<xref ref-type="bibr" rid="ref20">20</xref>-<xref ref-type="bibr" rid="ref22">22</xref>]</td>
                    <td>Engagement and buy-in fostered through participatory design and user (acceptance) testing; co-design of service model to identify gaps between new and existing workflows and foster collaborative problem-solving approaches to customization and configuration of the technology-enabled solution for the benefit of service quality improvement; development of a communication strategy to assist the service with messaging within internal stakeholders and service users</td>
                  </tr>
                  <tr valign="top">
                    <td>Staff turnover and lack of staff resources [<xref ref-type="bibr" rid="ref17">17</xref>,<xref ref-type="bibr" rid="ref18">18</xref>]</td>
                    <td>Implementation Officer embedded in service to maintain continuity of support; contingency plan for training new staff</td>
                  </tr>
                  <tr valign="top">
                    <td>Innovation not able to adapt over time to meet staff needs [<xref ref-type="bibr" rid="ref17">17</xref>,<xref ref-type="bibr" rid="ref20">20</xref>,<xref ref-type="bibr" rid="ref22">22</xref>]</td>
                    <td>Ongoing evaluation of the technology-enabled solution; continuous and iterative refinement of the technology-enabled solution and service model throughout Phases 3 and 4 (implementation and sustainment)</td>
                  </tr>
                  <tr valign="top">
                    <td>Design and usability of the technology-enabled solution [<xref ref-type="bibr" rid="ref20">20</xref>,<xref ref-type="bibr" rid="ref21">21</xref>,<xref ref-type="bibr" rid="ref24">24</xref>]; adaptability/flexibility of the technology-enabled solution [<xref ref-type="bibr" rid="ref17">17</xref>,<xref ref-type="bibr" rid="ref18">18</xref>,<xref ref-type="bibr" rid="ref22">22</xref>]; compatibility/fit of technology-enabled solution with service mission [<xref ref-type="bibr" rid="ref17">17</xref>,<xref ref-type="bibr" rid="ref18">18</xref>]; user resistance to the technology-enabled solution [<xref ref-type="bibr" rid="ref22">22</xref>]; noninteroperability (or limited) with other information and communications technology systems [<xref ref-type="bibr" rid="ref17">17</xref>,<xref ref-type="bibr" rid="ref20">20</xref>]; fidelity of implementation [<xref ref-type="bibr" rid="ref17">17</xref>,<xref ref-type="bibr" rid="ref22">22</xref>]; availability of and user familiarity with required equipment to use the technology [<xref ref-type="bibr" rid="ref21">21</xref>,<xref ref-type="bibr" rid="ref24">24</xref>]</td>
                    <td>Service provider readiness assessment to determine compatibility of the technology-enabled solution with the service; co-design for a service-specific technology-enabled solution; technology configuration and customization arising from the co-design process, including co-design and co-development of service-specific content, as well as integration with service information and communications technology systems; iterative user experience and user acceptance testing; iterative evaluative processes related to technology and implementation process, highlighting adaptability of the technology; provision of ongoing education and training and technical assistance</td>
                  </tr>
                </tbody>
              </table>
            </table-wrap>
          </sec>
        </sec>
        <sec>
          <title>Phase 3: Implementation</title>
          <p>Phases 1 and 2 are seen as critical to successful implementation, as they foster buy-in and commitment from stakeholders to the principles of co-design and quality improvement, and they should lead to a point where all stakeholders feel heard and are able to participate in the change process. As far as possible, all known barriers and facilitators to implementation, as they apply to the local context, have been identified at this stage, with a mitigation strategy identified for each. Despite this, new barriers and facilitators may be uncovered, which were previously unknown or underestimated; thus, having clearly established processes and identified persons to support the service through the change is key to address issues as they arise. Providing on-the-ground support to service users and staff is valuable in the early weeks or months of implementation. Furthermore, gathering feedback from all user groups, as they use the technology-enabled solution in practice, refines the technology and service model, but more importantly, the workforce and structural changes required to improve service quality (<xref rid="figure2" ref-type="fig">Figure 2</xref>). As illustrated in <xref rid="figure1" ref-type="fig">Figure 1</xref>, ongoing feedback from service staff (eg, health professionals, administrators, and service managers) is collected via Web-based surveys, semistructured interviews, and workshops to evaluate and monitor the impact of embedding the technology-enabled solution in the service, including (1) service-level changes in outcomes and processes, as well (2) as the digital readiness and competence of service staff, (3) quality, usability, and acceptability of the solution, and (4) social return on investment of embedding the solution in the service. In addition, feedback from both staff and consumers about existing and newly designed functionality is captured through quarterly user testing sessions. The fortnightly Implementation Officer Logs are also used to gather data from service staff, including anonymous commentary and feedback provided by individuals with lived experience, including consumers and their families or supportive others, who are engaging with the solution as part of care, which are then fed back through to R&amp;D processes to inform the iterative design and development of the technology-enabled solution. This feedback may include technical difficulties, as well as comments in relation to the user experience and clinical aspects of the solution. Google Analytics is also embedded within the solution to allow back-end analysis of user behavior, including details regarding the features of solution that consumers use most consistently, as well as those features with which they disengage most quickly. Of note, in the absence of a 24-hour monitoring protocol to ensure the safety of all users at this time, there is no option for consumers to provide free text feedback.</p>
          <p>Importantly, it is generally accepted in digital mental health research that the technology-enabled solution will be iteratively developed, designed, and refined during implementation. The expected outcome of Phase 3 is for the technology to be embedded and integrated within the service, such that it is seen as a vital piece of standard care, enabling and maintaining ongoing service quality improvement and reform.</p>
        </sec>
        <sec>
          <title>Phase 4: Sustainment and Scalability</title>
          <p>In accordance with Lennox et al [<xref ref-type="bibr" rid="ref39">39</xref>], the primary objective of Phase 4 is the continuation and maintenance of our technology-enabled solutions and their associated outcomes within a health service, as well as the iterative process of evaluation and design, to address problems and emerging needs and demands of the service, individual populations, and the broader context [<xref ref-type="bibr" rid="ref15">15</xref>]. To achieve this aim, prospective approaches are employed throughout the preceding phases to build relationships with and foster buy-in from key stakeholders (ie, consumers with lived experience and their families, health professionals, administrators, and service managers), as well as to iteratively design and refine the technology-enabled solution to adapt to the changing needs of the stakeholders and service [<xref ref-type="bibr" rid="ref39">39</xref>]. Employing these processes helps to ensure the continued effectiveness of the technology-enabled solution, including improved access to care and resources to promote mental health and well-being, the integration of the technology within the service, and community ownership of the solution [<xref ref-type="bibr" rid="ref40">40</xref>]. When taken together, this reflects the primary outcome of Phase 4, namely technology-enabled mental health services reform.</p>
          <p>Furthermore, the objective of Phase 4 is to leave a configurable technology-enabled solution in place for ongoing and continued benefit to the service, following the replacement of a locally available Implementation Officer with a more sustainable supporting resource that is readily available remotely.</p>
        </sec>
      </sec>
      <sec>
        <title>Participating Centers</title>
        <p>At the time of this publication, the implementation science protocol had gone live in the following participating centers: headspace services (Ashfield, Camperdown, Coffs Harbour, Hurstville, Lismore, Port Macquarie, Miranda, and Tweeds Head, New South Wales, and Edinburgh North, South Australia), Butterfly’s National Helpline, and Open Arms—Veterans and Families Counselling Surry Hills, New South Wales.</p>
      </sec>
      <sec>
        <title>Sample Size</title>
        <p>The protocol phases do not have an upper or lower limit on the number of participants, as this will vary by participating center, both in relation to the number of staff members (eg, health professionals, administrators, and service managers) and the size and diversity of their consumer base.</p>
      </sec>
      <sec>
        <title>Data Analyses</title>
        <p>Qualitative and quantitative data analyses will be conducted to assess the success of the implementation at the level of the consumer, health professional, and service, and, where possible, comparative analyses will be run between and within participating centers and populations, to allow for the identification of commonalities and differences in outcomes.</p>
      </sec>
      <sec>
        <title>Ethics</title>
        <p>Ethics approvals to conduct all aspects of the protocol are sought from the relevant governing Human Research Ethics Committees for the participating centers.</p>
      </sec>
    </sec>
    <sec sec-type="results">
      <title>Results</title>
      <p>At the time of this publication, all participating centers were in Phase 3. The first results from Phases 1 to 3 are expected to be submitted for publication in 2020, with Phase 4 data expected thereafter.</p>
    </sec>
    <sec sec-type="discussion">
      <title>Discussion</title>
      <p>The international goal of substantially improving the quality of mental health services is central to many technology-based innovation implementation efforts in mental health service delivery. As seen in a range of other industries, technology-enabled disruption brings with it significant changes to conventional practice and experience for all stakeholders. The greater the gap between the innovation and conventional practice, the greater the implementation challenge and, arguably, the greater the need for technology and service co-design with all stakeholders. This protocol incorporates the findings that affect implementation success in the rapidly evolving implementation science literature, to serve as both preemptive mitigation strategies and foci for surveillance throughout each of the implementation phases. With the aim of avoiding obsolescence of the solutions, our implementation science protocol also stresses the parallel and iterative evaluation of the effectiveness of the technology-enabled solution alongside the success, or lack thereof, of the implementation.</p>
    </sec>
  </body>
  <back>
    <app-group/>
    <glossary>
      <title>Abbreviations</title>
      <def-list>
        <def-item>
          <term id="abb1">BMC</term>
          <def>
            <p>Brain and Mind Centre</p>
          </def>
        </def-item>
        <def-item>
          <term id="abb2">HIT</term>
          <def>
            <p>health information technology</p>
          </def>
        </def-item>
        <def-item>
          <term id="abb3">ICT</term>
          <def>
            <p>information and communications technology</p>
          </def>
        </def-item>
        <def-item>
          <term id="abb4">R&amp;D</term>
          <def>
            <p>research and development</p>
          </def>
        </def-item>
      </def-list>
    </glossary>
    <ack>
      <p>The authors wish to acknowledge the consumers, health professionals, and supportive others who consented to participate in the research that has informed the development of the implementation science protocol. On June 30, 2017, the Australian Government Department of Health and InnoWell (a joint venture between the University of Sydney and PwC [Australia]) entered into a 3-year Funding Agreement to the value of Aus $30 million. This Funding Agreement provides for a series of collaborative research studies to design, develop, build, and evaluate technology-enabled solutions (using the InnoWell Platform) in four populations across the lifespan, including young children and their families, young people, adults, and healthy older adults. Beyond June 2020, InnoWell will support the R&amp;D work financially, as it represents the core work necessary to continuously and iteratively evolve the technology-enabled solution, and its role in redesigning health care aligned to key reform priorities, including person-centered care, shared decision making, staged care, and data-driven clinical decision making through routine outcome monitoring.</p>
    </ack>
    <fn-group>
      <fn fn-type="con">
        <p>IBH, TAD, and SC were integral in securing funding to support the study. All authors contributed to the development of the implementation science protocol. HML prepared the first draft of the manuscript. All authors contributed to and approved the final manuscript.</p>
      </fn>
      <fn fn-type="conflict">
        <p>IBH was an inaugural Commissioner on Australia’s National Mental Health Commission (2012-18). He is the Co-Director, Health and Policy at the Brain and Mind Centre (BMC), the University of Sydney. The BMC operates an early-intervention youth service at Camperdown, under contract to <italic>headspace</italic>. IBH has previously led community-based and pharmaceutical industry–supported (Wyeth, Eli Lily, Servier, Pfizer, and AstraZeneca) projects focused on the identification and better management of anxiety and depression. He was a member of the Medical Advisory Panel for Medibank Private until October 2017, a Board Member of Psychosis Australia Trust, and a member of Veterans Mental Health Clinical Reference group. He is the Chief Scientific Advisor to, and an equity shareholder in, InnoWell. InnoWell has been formed by the University of Sydney and PwC (Australia) to deliver the Aus $30 million Australian Government-funded <italic>Project Synergy</italic>. Project Synergy is a 3-year program for the transformation of mental health services through the use of innovative technologies. The other authors have no conflicts to declare.</p>
      </fn>
    </fn-group>
    <ref-list>
      <ref id="ref1">
        <label>1</label>
        <nlm-citation citation-type="web">
          <source>iTunes - Apple</source>
          <year>2016</year>
          <access-date>2019-08-08</access-date>
          <comment>MoodMission App 
          <ext-link ext-link-type="uri" xlink:type="simple" xlink:href="https://apps.apple.com/us/app/moodmission/id1140332763">https://apps.apple.com/us/app/moodmission/id1140332763</ext-link></comment>
        </nlm-citation>
      </ref>
      <ref id="ref2">
        <label>2</label>
        <nlm-citation citation-type="web">
          <source>iTunes - Apple</source>
          <year>2013</year>
          <access-date>2019-08-08</access-date>
          <comment>CBT-i Coach App 
          <ext-link ext-link-type="uri" xlink:type="simple" xlink:href="https://apps.apple.com/us/app/cbt-i-coach/id655918660">https://apps.apple.com/us/app/cbt-i-coach/id655918660</ext-link></comment>
        </nlm-citation>
      </ref>
      <ref id="ref3">
        <label>3</label>
        <nlm-citation citation-type="journal">
          <person-group person-group-type="author">
            <name name-style="western">
              <surname>Glasgow</surname>
              <given-names>RE</given-names>
            </name>
            <name name-style="western">
              <surname>Lichtenstein</surname>
              <given-names>E</given-names>
            </name>
            <name name-style="western">
              <surname>Marcus</surname>
              <given-names>AC</given-names>
            </name>
          </person-group>
          <article-title>Why don't we see more translation of health promotion research to practice? Rethinking the efficacy-to-effectiveness transition</article-title>
          <source>Am J Public Health</source>
          <year>2003</year>
          <month>08</month>
          <volume>93</volume>
          <issue>8</issue>
          <fpage>1261</fpage>
          <lpage>7</lpage>
          <pub-id pub-id-type="doi">10.2105/ajph.93.8.1261</pub-id>
          <pub-id pub-id-type="medline">12893608</pub-id>
          <pub-id pub-id-type="pmcid">PMC1447950</pub-id>
        </nlm-citation>
      </ref>
      <ref id="ref4">
        <label>4</label>
        <nlm-citation citation-type="journal">
          <person-group person-group-type="author">
            <name name-style="western">
              <surname>Greenhalgh</surname>
              <given-names>T</given-names>
            </name>
            <name name-style="western">
              <surname>A'Court</surname>
              <given-names>C</given-names>
            </name>
            <name name-style="western">
              <surname>Shaw</surname>
              <given-names>S</given-names>
            </name>
          </person-group>
          <article-title>Understanding heart failure; explaining telehealth - a hermeneutic systematic review</article-title>
          <source>BMC Cardiovasc Disord</source>
          <year>2017</year>
          <month>06</month>
          <day>14</day>
          <volume>17</volume>
          <issue>1</issue>
          <fpage>156</fpage>
          <comment>
            <ext-link ext-link-type="uri" xlink:type="simple" xlink:href="https://bmccardiovascdisord.biomedcentral.com/articles/10.1186/s12872-017-0594-2"/>
          </comment>
          <pub-id pub-id-type="doi">10.1186/s12872-017-0594-2</pub-id>
          <pub-id pub-id-type="medline">28615004</pub-id>
          <pub-id pub-id-type="pii">10.1186/s12872-017-0594-2</pub-id>
          <pub-id pub-id-type="pmcid">PMC5471857</pub-id>
        </nlm-citation>
      </ref>
      <ref id="ref5">
        <label>5</label>
        <nlm-citation citation-type="journal">
          <person-group person-group-type="author">
            <name name-style="western">
              <surname>de Bont</surname>
              <given-names>A</given-names>
            </name>
            <name name-style="western">
              <surname>Bal</surname>
              <given-names>R</given-names>
            </name>
          </person-group>
          <article-title>Telemedicine in interdisciplinary work practices: on an IT system that met the criteria for success set out by its sponsors, yet failed to become part of every-day clinical routines</article-title>
          <source>BMC Med Inform Decis Mak</source>
          <year>2008</year>
          <month>10</month>
          <day>27</day>
          <volume>8</volume>
          <fpage>47</fpage>
          <comment>
            <ext-link ext-link-type="uri" xlink:type="simple" xlink:href="https://bmcmedinformdecismak.biomedcentral.com/articles/10.1186/1472-6947-8-47"/>
          </comment>
          <pub-id pub-id-type="doi">10.1186/1472-6947-8-47</pub-id>
          <pub-id pub-id-type="medline">18954428</pub-id>
          <pub-id pub-id-type="pii">1472-6947-8-47</pub-id>
          <pub-id pub-id-type="pmcid">PMC2615749</pub-id>
        </nlm-citation>
      </ref>
      <ref id="ref6">
        <label>6</label>
        <nlm-citation citation-type="journal">
          <person-group person-group-type="author">
            <name name-style="western">
              <surname>Eccles</surname>
              <given-names>MP</given-names>
            </name>
            <name name-style="western">
              <surname>Mittman</surname>
              <given-names>BS</given-names>
            </name>
          </person-group>
          <article-title>Welcome to implementation science</article-title>
          <source>Implement Sci</source>
          <year>2006</year>
          <month>02</month>
          <day>22</day>
          <volume>1</volume>
          <issue>1</issue>
          <pub-id pub-id-type="doi">10.1186/1748-5908-1-1</pub-id>
        </nlm-citation>
      </ref>
      <ref id="ref7">
        <label>7</label>
        <nlm-citation citation-type="web">
          <source>Centers for Disease Control and Prevention</source>
          <year>2012</year>
          <access-date>2019-05-10</access-date>
          <comment>Division of Global HIV/AIDS: Implementation Science 
          <ext-link ext-link-type="uri" xlink:type="simple" xlink:href="https://www.cdc.gov/globalaids/What-CDC-is-Doing/implementation-science.html">https://www.cdc.gov/globalaids/What-CDC-is-Doing/implementation-science.html</ext-link>
          <ext-link ext-link-type="webcite" xlink:href="78FqWzryL"/></comment>
        </nlm-citation>
      </ref>
      <ref id="ref8">
        <label>8</label>
        <nlm-citation citation-type="web">
          <source>National Health and Medical Research Council</source>
          <year>2000</year>
          <access-date>2019-08-08</access-date>
          <comment>How to Put the Evidence Into Practice: Implementation and Dissemination Strategies: Handbook Series on Preparing Clinical Practice Guidelines 
          <ext-link ext-link-type="uri" xlink:type="simple" xlink:href="https://www.nhmrc.gov.au/sites/default/files/2018-03/how-to-put-the-evidence-into-practice-implementation-and-dissemination.pdf">https://www.nhmrc.gov.au/sites/default/files/2018-03/how-to-put-the-evidence-into-practice-implementation-and-dissemination.pdf</ext-link></comment>
        </nlm-citation>
      </ref>
      <ref id="ref9">
        <label>9</label>
        <nlm-citation citation-type="web">
          <source>The National Institute of Mental Health</source>
          <year>2018</year>
          <access-date>2019-05-10</access-date>
          <comment>Dissemination and Implementation Research Program 
          <ext-link ext-link-type="uri" xlink:type="simple" xlink:href="https://tinyurl.com/y57vh3o2">https://tinyurl.com/y57vh3o2</ext-link>
          <ext-link ext-link-type="webcite" xlink:href="78FqqjRhf"/></comment>
        </nlm-citation>
      </ref>
      <ref id="ref10">
        <label>10</label>
        <nlm-citation citation-type="journal">
          <person-group person-group-type="author">
            <name name-style="western">
              <surname>Nilsen</surname>
              <given-names>P</given-names>
            </name>
          </person-group>
          <article-title>Making sense of implementation theories, models and frameworks</article-title>
          <source>Implement Sci</source>
          <year>2015</year>
          <month>04</month>
          <day>21</day>
          <volume>10</volume>
          <fpage>53</fpage>
          <comment>
            <ext-link ext-link-type="uri" xlink:type="simple" xlink:href="https://implementationscience.biomedcentral.com/articles/10.1186/s13012-015-0242-0"/>
          </comment>
          <pub-id pub-id-type="doi">10.1186/s13012-015-0242-0</pub-id>
          <pub-id pub-id-type="medline">25895742</pub-id>
          <pub-id pub-id-type="pii">10.1186/s13012-015-0242-0</pub-id>
          <pub-id pub-id-type="pmcid">PMC4406164</pub-id>
        </nlm-citation>
      </ref>
      <ref id="ref11">
        <label>11</label>
        <nlm-citation citation-type="journal">
          <person-group person-group-type="author">
            <name name-style="western">
              <surname>Chaudhry</surname>
              <given-names>B</given-names>
            </name>
            <name name-style="western">
              <surname>Wang</surname>
              <given-names>J</given-names>
            </name>
            <name name-style="western">
              <surname>Wu</surname>
              <given-names>S</given-names>
            </name>
            <name name-style="western">
              <surname>Maglione</surname>
              <given-names>M</given-names>
            </name>
            <name name-style="western">
              <surname>Mojica</surname>
              <given-names>W</given-names>
            </name>
            <name name-style="western">
              <surname>Roth</surname>
              <given-names>E</given-names>
            </name>
            <name name-style="western">
              <surname>Morton</surname>
              <given-names>SC</given-names>
            </name>
            <name name-style="western">
              <surname>Shekelle</surname>
              <given-names>PG</given-names>
            </name>
          </person-group>
          <article-title>Systematic review: impact of health information technology on quality, efficiency, and costs of medical care</article-title>
          <source>Ann Intern Med</source>
          <year>2006</year>
          <month>05</month>
          <day>16</day>
          <volume>144</volume>
          <issue>10</issue>
          <fpage>742</fpage>
          <lpage>52</lpage>
          <pub-id pub-id-type="doi">10.7326/0003-4819-144-10-200605160-00125</pub-id>
          <pub-id pub-id-type="medline">16702590</pub-id>
          <pub-id pub-id-type="pii">0000605-200605160-00125</pub-id>
        </nlm-citation>
      </ref>
      <ref id="ref12">
        <label>12</label>
        <nlm-citation citation-type="journal">
          <person-group person-group-type="author">
            <name name-style="western">
              <surname>Lawes-Wickwar</surname>
              <given-names>S</given-names>
            </name>
            <name name-style="western">
              <surname>McBain</surname>
              <given-names>H</given-names>
            </name>
            <name name-style="western">
              <surname>Mulligan</surname>
              <given-names>K</given-names>
            </name>
          </person-group>
          <article-title>Application and effectiveness of telehealth to support severe mental illness management: systematic review</article-title>
          <source>JMIR Ment Health</source>
          <year>2018</year>
          <month>11</month>
          <day>21</day>
          <volume>5</volume>
          <issue>4</issue>
          <fpage>e62</fpage>
          <pub-id pub-id-type="doi">10.2196/mental.8816</pub-id>
          <pub-id pub-id-type="medline">30463836</pub-id>
          <pub-id pub-id-type="pii">v5i4e62</pub-id>
          <pub-id pub-id-type="pmcid">PMC6314801</pub-id>
        </nlm-citation>
      </ref>
      <ref id="ref13">
        <label>13</label>
        <nlm-citation citation-type="journal">
          <person-group person-group-type="author">
            <name name-style="western">
              <surname>Feijt</surname>
              <given-names>MA</given-names>
            </name>
            <name name-style="western">
              <surname>de Kort</surname>
              <given-names>YA</given-names>
            </name>
            <name name-style="western">
              <surname>Bongers</surname>
              <given-names>IM</given-names>
            </name>
            <name name-style="western">
              <surname>IJsselsteijn</surname>
              <given-names>WA</given-names>
            </name>
          </person-group>
          <article-title>Perceived drivers and barriers to the adoption of emental health by psychologists: the construction of the levels of adoption of emental health model</article-title>
          <source>J Med Internet Res</source>
          <year>2018</year>
          <volume>20</volume>
          <issue>4</issue>
          <fpage>e153</fpage>
          <pub-id pub-id-type="doi">10.2196/jmir.9485</pub-id>
          <pub-id pub-id-type="medline">29691215</pub-id>
        </nlm-citation>
      </ref>
      <ref id="ref14">
        <label>14</label>
        <nlm-citation citation-type="journal">
          <person-group person-group-type="author">
            <name name-style="western">
              <surname>Ross</surname>
              <given-names>J</given-names>
            </name>
            <name name-style="western">
              <surname>Stevenson</surname>
              <given-names>F</given-names>
            </name>
            <name name-style="western">
              <surname>Lau</surname>
              <given-names>R</given-names>
            </name>
            <name name-style="western">
              <surname>Murray</surname>
              <given-names>E</given-names>
            </name>
          </person-group>
          <article-title>Exploring the challenges of implementing e-health: a protocol for an update of a systematic review of reviews</article-title>
          <source>BMJ Open</source>
          <year>2015</year>
          <volume>5</volume>
          <issue>4</issue>
          <fpage>e006773</fpage>
          <pub-id pub-id-type="doi">10.1136/bmjopen-2014-006773</pub-id>
          <pub-id pub-id-type="medline">25869685</pub-id>
        </nlm-citation>
      </ref>
      <ref id="ref15">
        <label>15</label>
        <nlm-citation citation-type="journal">
          <person-group person-group-type="author">
            <name name-style="western">
              <surname>Mohr</surname>
              <given-names>DC</given-names>
            </name>
            <name name-style="western">
              <surname>Lyon</surname>
              <given-names>AR</given-names>
            </name>
            <name name-style="western">
              <surname>Lattie</surname>
              <given-names>EG</given-names>
            </name>
            <name name-style="western">
              <surname>Reddy</surname>
              <given-names>M</given-names>
            </name>
            <name name-style="western">
              <surname>Schueller</surname>
              <given-names>SM</given-names>
            </name>
          </person-group>
          <article-title>Accelerating digital mental health research from early design and creation to successful implementation and sustainment</article-title>
          <source>J Med Internet Res</source>
          <year>2017</year>
          <volume>19</volume>
          <issue>5</issue>
          <fpage>e153</fpage>
          <pub-id pub-id-type="doi">10.2196/jmir.7725</pub-id>
          <pub-id pub-id-type="medline">28490417</pub-id>
        </nlm-citation>
      </ref>
      <ref id="ref16">
        <label>16</label>
        <nlm-citation citation-type="journal">
          <person-group person-group-type="author">
            <name name-style="western">
              <surname>Hermes</surname>
              <given-names>ED</given-names>
            </name>
            <name name-style="western">
              <surname>Lyon</surname>
              <given-names>AR</given-names>
            </name>
            <name name-style="western">
              <surname>Schueller</surname>
              <given-names>SM</given-names>
            </name>
            <name name-style="western">
              <surname>Glass</surname>
              <given-names>JE</given-names>
            </name>
          </person-group>
          <article-title>Measuring the implementation of behavioral intervention technologies: recharacterization of established outcomes</article-title>
          <source>J Med Internet Res</source>
          <year>2019</year>
          <month>01</month>
          <day>25</day>
          <volume>21</volume>
          <issue>1</issue>
          <fpage>e11752</fpage>
          <comment>
            <ext-link ext-link-type="uri" xlink:type="simple" xlink:href="https://www.jmir.org/2019/1/e11752/"/>
          </comment>
          <pub-id pub-id-type="doi">10.2196/11752</pub-id>
          <pub-id pub-id-type="medline">30681966</pub-id>
          <pub-id pub-id-type="pii">v21i1e11752</pub-id>
          <pub-id pub-id-type="pmcid">PMC6367669</pub-id>
        </nlm-citation>
      </ref>
      <ref id="ref17">
        <label>17</label>
        <nlm-citation citation-type="journal">
          <person-group person-group-type="author">
            <name name-style="western">
              <surname>Bach-Mortensen</surname>
              <given-names>AM</given-names>
            </name>
            <name name-style="western">
              <surname>Lange</surname>
              <given-names>BC</given-names>
            </name>
            <name name-style="western">
              <surname>Montgomery</surname>
              <given-names>P</given-names>
            </name>
          </person-group>
          <article-title>Barriers and facilitators to implementing evidence-based interventions among third sector organisations: a systematic review</article-title>
          <source>Implement Sci</source>
          <year>2018</year>
          <month>07</month>
          <day>30</day>
          <volume>13</volume>
          <issue>1</issue>
          <fpage>103</fpage>
          <comment>
            <ext-link ext-link-type="uri" xlink:type="simple" xlink:href="https://implementationscience.biomedcentral.com/articles/10.1186/s13012-018-0789-7"/>
          </comment>
          <pub-id pub-id-type="doi">10.1186/s13012-018-0789-7</pub-id>
          <pub-id pub-id-type="medline">30060744</pub-id>
          <pub-id pub-id-type="pii">10.1186/s13012-018-0789-7</pub-id>
          <pub-id pub-id-type="pmcid">PMC6065156</pub-id>
        </nlm-citation>
      </ref>
      <ref id="ref18">
        <label>18</label>
        <nlm-citation citation-type="journal">
          <person-group person-group-type="author">
            <name name-style="western">
              <surname>Durlak</surname>
              <given-names>JA</given-names>
            </name>
            <name name-style="western">
              <surname>DuPre</surname>
              <given-names>EP</given-names>
            </name>
          </person-group>
          <article-title>Implementation matters: a review of research on the influence of implementation on program outcomes and the factors affecting implementation</article-title>
          <source>Am J Community Psychol</source>
          <year>2008</year>
          <month>06</month>
          <volume>41</volume>
          <issue>3-4</issue>
          <fpage>327</fpage>
          <lpage>50</lpage>
          <pub-id pub-id-type="doi">10.1007/s10464-008-9165-0</pub-id>
          <pub-id pub-id-type="medline">18322790</pub-id>
        </nlm-citation>
      </ref>
      <ref id="ref19">
        <label>19</label>
        <nlm-citation citation-type="journal">
          <person-group person-group-type="author">
            <name name-style="western">
              <surname>Geerligs</surname>
              <given-names>L</given-names>
            </name>
            <name name-style="western">
              <surname>Rankin</surname>
              <given-names>NM</given-names>
            </name>
            <name name-style="western">
              <surname>Shepherd</surname>
              <given-names>HL</given-names>
            </name>
            <name name-style="western">
              <surname>Butow</surname>
              <given-names>P</given-names>
            </name>
          </person-group>
          <article-title>Hospital-based interventions: a systematic review of staff-reported barriers and facilitators to implementation processes</article-title>
          <source>Implement Sci</source>
          <year>2018</year>
          <month>02</month>
          <day>23</day>
          <volume>13</volume>
          <issue>1</issue>
          <fpage>36</fpage>
          <comment>
            <ext-link ext-link-type="uri" xlink:type="simple" xlink:href="https://implementationscience.biomedcentral.com/articles/10.1186/s13012-018-0726-9"/>
          </comment>
          <pub-id pub-id-type="doi">10.1186/s13012-018-0726-9</pub-id>
          <pub-id pub-id-type="medline">29475440</pub-id>
          <pub-id pub-id-type="pii">10.1186/s13012-018-0726-9</pub-id>
          <pub-id pub-id-type="pmcid">PMC5824580</pub-id>
        </nlm-citation>
      </ref>
      <ref id="ref20">
        <label>20</label>
        <nlm-citation citation-type="journal">
          <person-group person-group-type="author">
            <name name-style="western">
              <surname>Granja</surname>
              <given-names>C</given-names>
            </name>
            <name name-style="western">
              <surname>Janssen</surname>
              <given-names>W</given-names>
            </name>
            <name name-style="western">
              <surname>Johansen</surname>
              <given-names>MA</given-names>
            </name>
          </person-group>
          <article-title>Factors determining the success and failure of ehealth interventions: systematic review of the literature</article-title>
          <source>J Med Internet Res</source>
          <year>2018</year>
          <month>05</month>
          <day>1</day>
          <volume>20</volume>
          <issue>5</issue>
          <fpage>e10235</fpage>
          <comment>
            <ext-link ext-link-type="uri" xlink:type="simple" xlink:href="https://www.jmir.org/2018/5/e10235/"/>
          </comment>
          <pub-id pub-id-type="doi">10.2196/10235</pub-id>
          <pub-id pub-id-type="medline">29716883</pub-id>
          <pub-id pub-id-type="pii">v20i5e10235</pub-id>
          <pub-id pub-id-type="pmcid">PMC5954232</pub-id>
        </nlm-citation>
      </ref>
      <ref id="ref21">
        <label>21</label>
        <nlm-citation citation-type="journal">
          <person-group person-group-type="author">
            <name name-style="western">
              <surname>Vis</surname>
              <given-names>C</given-names>
            </name>
            <name name-style="western">
              <surname>Mol</surname>
              <given-names>M</given-names>
            </name>
            <name name-style="western">
              <surname>Kleiboer</surname>
              <given-names>A</given-names>
            </name>
            <name name-style="western">
              <surname>Bührmann</surname>
              <given-names>L</given-names>
            </name>
            <name name-style="western">
              <surname>Finch</surname>
              <given-names>T</given-names>
            </name>
            <name name-style="western">
              <surname>Smit</surname>
              <given-names>J</given-names>
            </name>
            <name name-style="western">
              <surname>Riper</surname>
              <given-names>H</given-names>
            </name>
          </person-group>
          <article-title>Improving implementation of emental health for mood disorders in routine practice: systematic review of barriers and facilitating factors</article-title>
          <source>JMIR Ment Health</source>
          <year>2018</year>
          <month>03</month>
          <day>16</day>
          <volume>5</volume>
          <issue>1</issue>
          <fpage>e20</fpage>
          <comment>
            <ext-link ext-link-type="uri" xlink:type="simple" xlink:href="https://mental.jmir.org/2018/1/e20/"/>
          </comment>
          <pub-id pub-id-type="doi">10.2196/mental.9769</pub-id>
          <pub-id pub-id-type="medline">29549072</pub-id>
          <pub-id pub-id-type="pii">v5i1e20</pub-id>
          <pub-id pub-id-type="pmcid">PMC5878369</pub-id>
        </nlm-citation>
      </ref>
      <ref id="ref22">
        <label>22</label>
        <nlm-citation citation-type="journal">
          <person-group person-group-type="author">
            <name name-style="western">
              <surname>Greenhalgh</surname>
              <given-names>T</given-names>
            </name>
            <name name-style="western">
              <surname>Wherton</surname>
              <given-names>J</given-names>
            </name>
            <name name-style="western">
              <surname>Papoutsi</surname>
              <given-names>C</given-names>
            </name>
            <name name-style="western">
              <surname>Lynch</surname>
              <given-names>J</given-names>
            </name>
            <name name-style="western">
              <surname>Hughes</surname>
              <given-names>G</given-names>
            </name>
            <name name-style="western">
              <surname>A'Court</surname>
              <given-names>C</given-names>
            </name>
            <name name-style="western">
              <surname>Hinder</surname>
              <given-names>S</given-names>
            </name>
            <name name-style="western">
              <surname>Fahy</surname>
              <given-names>N</given-names>
            </name>
            <name name-style="western">
              <surname>Procter</surname>
              <given-names>R</given-names>
            </name>
            <name name-style="western">
              <surname>Shaw</surname>
              <given-names>S</given-names>
            </name>
          </person-group>
          <article-title>Beyond adoption: a new framework for theorizing and evaluating nonadoption, abandonment, and challenges to the scale-up, spread, and sustainability of health and care technologies</article-title>
          <source>J Med Internet Res</source>
          <year>2017</year>
          <month>11</month>
          <day>1</day>
          <volume>19</volume>
          <issue>11</issue>
          <fpage>e367</fpage>
          <comment>
            <ext-link ext-link-type="uri" xlink:type="simple" xlink:href="https://www.jmir.org/2017/11/e367/"/>
          </comment>
          <pub-id pub-id-type="doi">10.2196/jmir.8775</pub-id>
          <pub-id pub-id-type="medline">29092808</pub-id>
          <pub-id pub-id-type="pii">v19i11e367</pub-id>
          <pub-id pub-id-type="pmcid">PMC5688245</pub-id>
        </nlm-citation>
      </ref>
      <ref id="ref23">
        <label>23</label>
        <nlm-citation citation-type="journal">
          <person-group person-group-type="author">
            <name name-style="western">
              <surname>Davenport</surname>
              <given-names>TA</given-names>
            </name>
            <name name-style="western">
              <surname>LaMonica</surname>
              <given-names>HM</given-names>
            </name>
            <name name-style="western">
              <surname>Whittle</surname>
              <given-names>L</given-names>
            </name>
            <name name-style="western">
              <surname>English</surname>
              <given-names>A</given-names>
            </name>
            <name name-style="western">
              <surname>Iorfino</surname>
              <given-names>F</given-names>
            </name>
            <name name-style="western">
              <surname>Cross</surname>
              <given-names>S</given-names>
            </name>
            <name name-style="western">
              <surname>Hickie</surname>
              <given-names>IB</given-names>
            </name>
          </person-group>
          <article-title>Validation of the InnoWell platform: protocol for a clinical trial</article-title>
          <source>JMIR Res Protoc</source>
          <year>2019</year>
          <month>05</month>
          <day>31</day>
          <volume>8</volume>
          <issue>5</issue>
          <fpage>e13955</fpage>
          <comment>
            <ext-link ext-link-type="uri" xlink:type="simple" xlink:href="https://www.researchprotocols.org/2019/5/e13955/"/>
          </comment>
          <pub-id pub-id-type="doi">10.2196/13955</pub-id>
          <pub-id pub-id-type="medline">31152524</pub-id>
          <pub-id pub-id-type="pii">v8i5e13955</pub-id>
        </nlm-citation>
      </ref>
      <ref id="ref24">
        <label>24</label>
        <nlm-citation citation-type="journal">
          <person-group person-group-type="author">
            <name name-style="western">
              <surname>Cross</surname>
              <given-names>S</given-names>
            </name>
            <name name-style="western">
              <surname>Piper</surname>
              <given-names>S</given-names>
            </name>
            <name name-style="western">
              <surname>Davenport</surname>
              <given-names>TA</given-names>
            </name>
            <name name-style="western">
              <surname>Milton</surname>
              <given-names>A</given-names>
            </name>
            <name name-style="western">
              <surname>Iorfino</surname>
              <given-names>F</given-names>
            </name>
            <name name-style="western">
              <surname>Ricci</surname>
              <given-names>C</given-names>
            </name>
            <name name-style="western">
              <surname>Ospina-Pinillos</surname>
              <given-names>L</given-names>
            </name>
            <name name-style="western">
              <surname>Whittle</surname>
              <given-names>L</given-names>
            </name>
            <name name-style="western">
              <surname>Hickie</surname>
              <given-names>IB</given-names>
            </name>
          </person-group>
          <article-title>Implementation trial of a prototypic e-clinic within youth mental health services: staff experiences and reported service quality improvements</article-title>
          <source>Med J Aust</source>
          <year>2019</year>
          <comment>(forthcoming)</comment>
        </nlm-citation>
      </ref>
      <ref id="ref25">
        <label>25</label>
        <nlm-citation citation-type="journal">
          <person-group person-group-type="author">
            <name name-style="western">
              <surname>Boswell</surname>
              <given-names>JF</given-names>
            </name>
            <name name-style="western">
              <surname>Kraus</surname>
              <given-names>DR</given-names>
            </name>
            <name name-style="western">
              <surname>Miller</surname>
              <given-names>SD</given-names>
            </name>
            <name name-style="western">
              <surname>Lambert</surname>
              <given-names>MJ</given-names>
            </name>
          </person-group>
          <article-title>Implementing routine outcome monitoring in clinical practice: benefits, challenges, and solutions</article-title>
          <source>Psychother Res</source>
          <year>2015</year>
          <volume>25</volume>
          <issue>1</issue>
          <fpage>6</fpage>
          <lpage>19</lpage>
          <pub-id pub-id-type="doi">10.1080/10503307.2013.817696</pub-id>
          <pub-id pub-id-type="medline">23885809</pub-id>
        </nlm-citation>
      </ref>
      <ref id="ref26">
        <label>26</label>
        <nlm-citation citation-type="journal">
          <person-group person-group-type="author">
            <name name-style="western">
              <surname>Nazi</surname>
              <given-names>KM</given-names>
            </name>
          </person-group>
          <article-title>The personal health record paradox: health care professionals' perspectives and the information ecology of personal health record systems in organizational and clinical settings</article-title>
          <source>J Med Internet Res</source>
          <year>2013</year>
          <month>04</month>
          <day>4</day>
          <volume>15</volume>
          <issue>4</issue>
          <fpage>e70</fpage>
          <comment>
            <ext-link ext-link-type="uri" xlink:type="simple" xlink:href="https://www.jmir.org/2013/4/e70/"/>
          </comment>
          <pub-id pub-id-type="doi">10.2196/jmir.2443</pub-id>
          <pub-id pub-id-type="medline">23557596</pub-id>
          <pub-id pub-id-type="pii">v15i4e70</pub-id>
          <pub-id pub-id-type="pmcid">PMC3636319</pub-id>
        </nlm-citation>
      </ref>
      <ref id="ref27">
        <label>27</label>
        <nlm-citation citation-type="journal">
          <person-group person-group-type="author">
            <name name-style="western">
              <surname>Ceptureanu</surname>
              <given-names>S</given-names>
            </name>
            <name name-style="western">
              <surname>Ceptureanu</surname>
              <given-names>E</given-names>
            </name>
            <name name-style="western">
              <surname>Luchian</surname>
              <given-names>C</given-names>
            </name>
            <name name-style="western">
              <surname>Luchian</surname>
              <given-names>I</given-names>
            </name>
          </person-group>
          <article-title>Community based programs sustainability. A multidimensional analysis of sustainability factors</article-title>
          <source>Sustainability</source>
          <year>2018</year>
          <month>03</month>
          <day>19</day>
          <volume>10</volume>
          <issue>3</issue>
          <fpage>870</fpage>
          <comment>
            <ext-link ext-link-type="uri" xlink:type="simple" xlink:href="https://doi.org/10.3390/su10030870"/>
          </comment>
          <pub-id pub-id-type="doi">10.3390/su10030870</pub-id>
        </nlm-citation>
      </ref>
      <ref id="ref28">
        <label>28</label>
        <nlm-citation citation-type="journal">
          <person-group person-group-type="author">
            <name name-style="western">
              <surname>Ben-Tovim</surname>
              <given-names>DI</given-names>
            </name>
            <name name-style="western">
              <surname>Dougherty</surname>
              <given-names>ML</given-names>
            </name>
            <name name-style="western">
              <surname>O'Connell</surname>
              <given-names>TJ</given-names>
            </name>
            <name name-style="western">
              <surname>McGrath</surname>
              <given-names>KM</given-names>
            </name>
          </person-group>
          <article-title>Patient journeys: the process of clinical redesign</article-title>
          <source>Med J Aust</source>
          <year>2008</year>
          <month>03</month>
          <day>17</day>
          <volume>188</volume>
          <issue>6 Suppl</issue>
          <fpage>S14</fpage>
          <lpage>7</lpage>
          <pub-id pub-id-type="doi">10.5694/j.1326-5377.2008.tb01668.x</pub-id>
          <pub-id pub-id-type="medline">18341470</pub-id>
          <pub-id pub-id-type="pii">ben11040_fm</pub-id>
        </nlm-citation>
      </ref>
      <ref id="ref29">
        <label>29</label>
        <nlm-citation citation-type="journal">
          <person-group person-group-type="author">
            <name name-style="western">
              <surname>Meyers</surname>
              <given-names>DC</given-names>
            </name>
            <name name-style="western">
              <surname>Durlak</surname>
              <given-names>JA</given-names>
            </name>
            <name name-style="western">
              <surname>Wandersman</surname>
              <given-names>A</given-names>
            </name>
          </person-group>
          <article-title>The quality implementation framework: a synthesis of critical steps in the implementation process</article-title>
          <source>Am J Community Psychol</source>
          <year>2012</year>
          <month>12</month>
          <volume>50</volume>
          <issue>3-4</issue>
          <fpage>462</fpage>
          <lpage>80</lpage>
          <pub-id pub-id-type="doi">10.1007/s10464-012-9522-x</pub-id>
          <pub-id pub-id-type="medline">22644083</pub-id>
        </nlm-citation>
      </ref>
      <ref id="ref30">
        <label>30</label>
        <nlm-citation citation-type="journal">
          <person-group person-group-type="author">
            <name name-style="western">
              <surname>Damschroder</surname>
              <given-names>LJ</given-names>
            </name>
            <name name-style="western">
              <surname>Aron</surname>
              <given-names>DC</given-names>
            </name>
            <name name-style="western">
              <surname>Keith</surname>
              <given-names>RE</given-names>
            </name>
            <name name-style="western">
              <surname>Kirsh</surname>
              <given-names>SR</given-names>
            </name>
            <name name-style="western">
              <surname>Alexander</surname>
              <given-names>JA</given-names>
            </name>
            <name name-style="western">
              <surname>Lowery</surname>
              <given-names>JC</given-names>
            </name>
          </person-group>
          <article-title>Fostering implementation of health services research findings into practice: a consolidated framework for advancing implementation science</article-title>
          <source>Implement Sci</source>
          <year>2009</year>
          <month>08</month>
          <day>7</day>
          <volume>4</volume>
          <fpage>50</fpage>
          <comment>
            <ext-link ext-link-type="uri" xlink:type="simple" xlink:href="https://implementationscience.biomedcentral.com/articles/10.1186/1748-5908-4-50"/>
          </comment>
          <pub-id pub-id-type="doi">10.1186/1748-5908-4-50</pub-id>
          <pub-id pub-id-type="medline">19664226</pub-id>
          <pub-id pub-id-type="pii">1748-5908-4-50</pub-id>
          <pub-id pub-id-type="pmcid">PMC2736161</pub-id>
        </nlm-citation>
      </ref>
      <ref id="ref31">
        <label>31</label>
        <nlm-citation citation-type="journal">
          <person-group person-group-type="author">
            <name name-style="western">
              <surname>Feldstein</surname>
              <given-names>AC</given-names>
            </name>
            <name name-style="western">
              <surname>Glasgow</surname>
              <given-names>RE</given-names>
            </name>
          </person-group>
          <article-title>A practical, robust implementation and sustainability model (PRISM) for integrating research findings into practice</article-title>
          <source>Jt Comm J Qual Patient Saf</source>
          <year>2008</year>
          <month>04</month>
          <volume>34</volume>
          <issue>4</issue>
          <fpage>228</fpage>
          <lpage>43</lpage>
          <pub-id pub-id-type="doi">10.1016/S1553-7250(08)34030-6</pub-id>
          <pub-id pub-id-type="medline">18468362</pub-id>
          <pub-id pub-id-type="pii">S1553-7250(08)34030-6</pub-id>
        </nlm-citation>
      </ref>
      <ref id="ref32">
        <label>32</label>
        <nlm-citation citation-type="journal">
          <person-group person-group-type="author">
            <name name-style="western">
              <surname>Gordon</surname>
              <given-names>NP</given-names>
            </name>
            <name name-style="western">
              <surname>Hornbrook</surname>
              <given-names>MC</given-names>
            </name>
          </person-group>
          <article-title>Differences in access to and preferences for using patient portals and other ehealth technologies based on race, ethnicity, and age: a database and survey study of seniors in a large health plan</article-title>
          <source>J Med Internet Res</source>
          <year>2016</year>
          <month>03</month>
          <day>4</day>
          <volume>18</volume>
          <issue>3</issue>
          <fpage>e50</fpage>
          <comment>
            <ext-link ext-link-type="uri" xlink:type="simple" xlink:href="https://www.jmir.org/2016/3/e50/"/>
          </comment>
          <pub-id pub-id-type="doi">10.2196/jmir.5105</pub-id>
          <pub-id pub-id-type="medline">26944212</pub-id>
          <pub-id pub-id-type="pii">v18i3e50</pub-id>
          <pub-id pub-id-type="pmcid">PMC4799429</pub-id>
        </nlm-citation>
      </ref>
      <ref id="ref33">
        <label>33</label>
        <nlm-citation citation-type="journal">
          <person-group person-group-type="author">
            <name name-style="western">
              <surname>van Dyk</surname>
              <given-names>L</given-names>
            </name>
          </person-group>
          <article-title>A review of telehealth service implementation frameworks</article-title>
          <source>Int J Environ Res Public Health</source>
          <year>2014</year>
          <month>01</month>
          <day>23</day>
          <volume>11</volume>
          <issue>2</issue>
          <fpage>1279</fpage>
          <lpage>98</lpage>
          <comment>
            <ext-link ext-link-type="uri" xlink:type="simple" xlink:href="http://www.mdpi.com/resolver?pii=ijerph110201279"/>
          </comment>
          <pub-id pub-id-type="doi">10.3390/ijerph110201279</pub-id>
          <pub-id pub-id-type="medline">24464237</pub-id>
          <pub-id pub-id-type="pii">ijerph110201279</pub-id>
          <pub-id pub-id-type="pmcid">PMC3945538</pub-id>
        </nlm-citation>
      </ref>
      <ref id="ref34">
        <label>34</label>
        <nlm-citation citation-type="journal">
          <person-group person-group-type="author">
            <name name-style="western">
              <surname>Meyers</surname>
              <given-names>DC</given-names>
            </name>
            <name name-style="western">
              <surname>Katz</surname>
              <given-names>J</given-names>
            </name>
            <name name-style="western">
              <surname>Chien</surname>
              <given-names>V</given-names>
            </name>
            <name name-style="western">
              <surname>Wandersman</surname>
              <given-names>A</given-names>
            </name>
            <name name-style="western">
              <surname>Scaccia</surname>
              <given-names>JP</given-names>
            </name>
            <name name-style="western">
              <surname>Wright</surname>
              <given-names>A</given-names>
            </name>
          </person-group>
          <article-title>Practical implementation science: developing and piloting the quality implementation tool</article-title>
          <source>Am J Community Psychol</source>
          <year>2012</year>
          <month>12</month>
          <volume>50</volume>
          <issue>3-4</issue>
          <fpage>481</fpage>
          <lpage>96</lpage>
          <pub-id pub-id-type="doi">10.1007/s10464-012-9521-y</pub-id>
          <pub-id pub-id-type="medline">22618025</pub-id>
        </nlm-citation>
      </ref>
      <ref id="ref35">
        <label>35</label>
        <nlm-citation citation-type="journal">
          <person-group person-group-type="author">
            <name name-style="western">
              <surname>Davenport</surname>
              <given-names>TA</given-names>
            </name>
            <name name-style="western">
              <surname>Milton</surname>
              <given-names>A</given-names>
            </name>
            <name name-style="western">
              <surname>Ospina-Pinillos</surname>
              <given-names>L</given-names>
            </name>
            <name name-style="western">
              <surname>Whittle</surname>
              <given-names>L</given-names>
            </name>
            <name name-style="western">
              <surname>Ricci</surname>
              <given-names>CS</given-names>
            </name>
            <name name-style="western">
              <surname>Burns</surname>
              <given-names>JM</given-names>
            </name>
            <name name-style="western">
              <surname>Hickie</surname>
              <given-names>IB</given-names>
            </name>
          </person-group>
          <article-title>The research and development cycle for project Synergy: an iterative process of participatory design, user testing and real-world trialing</article-title>
          <source>Med J Aust</source>
          <year>2019</year>
          <comment>(forthcoming)</comment>
        </nlm-citation>
      </ref>
      <ref id="ref36">
        <label>36</label>
        <nlm-citation citation-type="journal">
          <person-group person-group-type="author">
            <name name-style="western">
              <surname>Ospina-Pinillos</surname>
              <given-names>L</given-names>
            </name>
            <name name-style="western">
              <surname>Davenport</surname>
              <given-names>TA</given-names>
            </name>
            <name name-style="western">
              <surname>Ricci</surname>
              <given-names>CS</given-names>
            </name>
            <name name-style="western">
              <surname>Milton</surname>
              <given-names>AC</given-names>
            </name>
            <name name-style="western">
              <surname>Scott</surname>
              <given-names>EM</given-names>
            </name>
            <name name-style="western">
              <surname>Hickie</surname>
              <given-names>IB</given-names>
            </name>
          </person-group>
          <article-title>Developing a mental health eclinic to improve access to and quality of mental health care for young people: using participatory design as research methodologies</article-title>
          <source>J Med Internet Res</source>
          <year>2018</year>
          <month>05</month>
          <day>28</day>
          <volume>20</volume>
          <issue>5</issue>
          <fpage>e188</fpage>
          <comment>
            <ext-link ext-link-type="uri" xlink:type="simple" xlink:href="https://www.jmir.org/2018/5/e188/"/>
          </comment>
          <pub-id pub-id-type="doi">10.2196/jmir.9716</pub-id>
          <pub-id pub-id-type="medline">29807878</pub-id>
          <pub-id pub-id-type="pii">v20i5e188</pub-id>
          <pub-id pub-id-type="pmcid">PMC5996175</pub-id>
        </nlm-citation>
      </ref>
      <ref id="ref37">
        <label>37</label>
        <nlm-citation citation-type="journal">
          <person-group person-group-type="author">
            <name name-style="western">
              <surname>LaMonica</surname>
              <given-names>HM</given-names>
            </name>
            <name name-style="western">
              <surname>Davenport</surname>
              <given-names>T</given-names>
            </name>
            <name name-style="western">
              <surname>Burns</surname>
              <given-names>J</given-names>
            </name>
            <name name-style="western">
              <surname>Cross</surname>
              <given-names>S</given-names>
            </name>
            <name name-style="western">
              <surname>Hodson</surname>
              <given-names>S</given-names>
            </name>
            <name name-style="western">
              <surname>Veitch</surname>
              <given-names>J</given-names>
            </name>
            <name name-style="western">
              <surname>Hickie</surname>
              <given-names>IB</given-names>
            </name>
          </person-group>
          <article-title>Mental health service reform for open arms–veterans and families counselling: a participatory design study</article-title>
          <source>JMIR Form Res</source>
          <year>2019</year>
          <comment>(forthcoming)(forthcoming)</comment>
        </nlm-citation>
      </ref>
      <ref id="ref38">
        <label>38</label>
        <nlm-citation citation-type="journal">
          <person-group person-group-type="author">
            <name name-style="western">
              <surname>Cross</surname>
              <given-names>S</given-names>
            </name>
            <name name-style="western">
              <surname>Davenport</surname>
              <given-names>TA</given-names>
            </name>
            <name name-style="western">
              <surname>LaMonica</surname>
              <given-names>HM</given-names>
            </name>
            <name name-style="western">
              <surname>Hickie</surname>
              <given-names>IB</given-names>
            </name>
          </person-group>
          <article-title>Potential of real-time and integrated clinical data to drive continuous quality improvement in youth mental health services</article-title>
          <source>Med J Aust</source>
          <year>2019</year>
          <comment>(forthcoming)(forthcoming)</comment>
        </nlm-citation>
      </ref>
      <ref id="ref39">
        <label>39</label>
        <nlm-citation citation-type="journal">
          <person-group person-group-type="author">
            <name name-style="western">
              <surname>Lennox</surname>
              <given-names>L</given-names>
            </name>
            <name name-style="western">
              <surname>Maher</surname>
              <given-names>L</given-names>
            </name>
            <name name-style="western">
              <surname>Reed</surname>
              <given-names>J</given-names>
            </name>
          </person-group>
          <article-title>Navigating the sustainability landscape: a systematic review of sustainability approaches in healthcare</article-title>
          <source>Implement Sci</source>
          <year>2018</year>
          <month>02</month>
          <day>9</day>
          <volume>13</volume>
          <issue>1</issue>
          <fpage>27</fpage>
          <comment>
            <ext-link ext-link-type="uri" xlink:type="simple" xlink:href="https://implementationscience.biomedcentral.com/articles/10.1186/s13012-017-0707-4"/>
          </comment>
          <pub-id pub-id-type="doi">10.1186/s13012-017-0707-4</pub-id>
          <pub-id pub-id-type="medline">29426341</pub-id>
          <pub-id pub-id-type="pii">10.1186/s13012-017-0707-4</pub-id>
          <pub-id pub-id-type="pmcid">PMC5810192</pub-id>
        </nlm-citation>
      </ref>
      <ref id="ref40">
        <label>40</label>
        <nlm-citation citation-type="web">
          <source>World Health Organization</source>
          <year>2004</year>
          <access-date>2019-05-10</access-date>
          <comment>Guidelines for Conducting An Evaluation of the Sustainability Of CDTI Projects 
          <ext-link ext-link-type="uri" xlink:type="simple" xlink:href="https://www.who.int/apoc/publications/guidelinesevalsustainabilitycorrectedversionsept04.pdf">https://www.who.int/apoc/publications/guidelinesevalsustainabilitycorrectedversionsept04.pdf</ext-link>
          <ext-link ext-link-type="webcite" xlink:href="78G6W6uTf"/></comment>
        </nlm-citation>
      </ref>
    </ref-list>
  </back>
</article>
