Coverage for pyriandx/cli.py: 78%

1# -*- coding: utf-8 -*-

2"""PierianDx API Client ::: API client CLI/SDK for PierianDx web services

4Usage:

5 pyriandx <command> [options] [<args>...]

7Command:

8 help Print help and exit

9 version Print version and exit

10 case Get a case from Case API

11 list List cases from Case API, optionally apply filters to limit results

12 create Accession a new case from given input JSON file

13 upload Upload case files for given Case ID

14 run Create sequencer run for given Case ID

15 job Create informatics job for given Case ID and Run ID

16 poll Poll informatics job status for given Case ID and Job ID

17 report Get a report for given Case ID

19(See 'pyriandx <command> help' for more information on a specific command)

21Options:

22 -b, --base_url=base_url Base URL.

23 -u, --username=username Required if PDX_USERNAME does not exist. Usually email address.

24 -p, --password=password Required if PDX_PASSWORD does not exist.

25 -i, --institution=institution Required if PDX_INSTITUTION does not exist.

26 -d, --debug Make output more verbose innit.

27 -t, --trace Make output more and more verbose innit.

29Environment variables:

30 PDX_USERNAME If defined, uses this as username for authenticating to PierianDx

31 PDX_PASSWORD If defined, uses this as password for authenticating to PierianDx

32 PDX_INSTITUTION If defined, uses this as institution for authenticating to PierianDx

33 PDX_BASE_URL If defined, uses this as base URL for PierianDx service

34"""

35import json

36import logging

37import os

38import sys

39import time

41import coloredlogs

42import verboselogs

43from docopt import docopt

45from pyriandx.client import Client

46from . import __version__

48logger = verboselogs.VerboseLogger(__name__)

49verboselogs.add_log_level(verboselogs.SPAM, 'TRACE')

50verboselogs.install()

52coloredlogs.DEFAULT_LOG_FORMAT = '%(asctime)s %(name)-12s \t %(levelname)-8s %(message)s'

53coloredlogs.DEFAULT_LEVEL_STYLES = dict(

54 spam=dict(color='green', faint=True),

55 debug=dict(color='green'),

56 verbose=dict(),

57 info=dict(),

58 notice=dict(color='magenta'),

59 warning=dict(color='yellow'),

60 success=dict(color='green', bold=True),

61 error=dict(color='red'),

62 critical=dict(color='red', bold=True),

63)

64coloredlogs.install()

66DEFAULT_BASE_URL = "https://app.uat.pieriandx.com/cgw-api/v2.0.0"

69def _help(msg):

70 print(msg)

71 sys.exit(0)

74def _die(msg):

75 print(__doc__)

76 logger.critical(f"{msg}.")

77 sys.exit(1)

80def _halt(msg, doc):

81 print(doc)

82 logger.critical(f"{msg}.")

83 sys.exit(1)

86def _build(global_args):

87 username = global_args.get('--username', None)

88 if username is None:

89 username = os.getenv('PDX_USER', None) # backward compatible

90 if username is None:

91 username = os.getenv('PDX_USERNAME', None)

92 assert username is not None, _die("Please provide username via -u flag or PDX_USERNAME environment variable")

94 pw = global_args.get('--password', None)

95 if pw is None:

96 pw = os.getenv('PDX_SECRET', None) # backward compatible

97 if pw is None:

98 pw = os.getenv('PDX_PASSWORD', None)

99 assert pw is not None, _die("Please provide password via -p flag or PDX_PASSWORD environment variable")

100

101 inst = global_args.get('--institution', None)

102 if inst is None:

103 inst = os.getenv('PDX_INSTITUTION', None)

104 assert inst is not None, _die("Please provide institution via -i flag or PDX_INSTITUTION environment variable")

105

106 base_url = global_args.get('--base_url', None)

107 if base_url is None:

108 base_url = os.getenv('PDX_BASE_URL', None)

109 if base_url is None:

110 base_url = DEFAULT_BASE_URL

111

112 if "uat" in base_url:

113 logger.warning(f"You are working on PierianDx CGW 'UAT' environment -- {base_url}")

114 else:

115 logger.notice(f"Your working PierianDx CGW environment is -- {base_url}")

116

117 return Client(email=username, key=pw, institution=inst, base_url=base_url)

118

119

120def _dispatch():

121 global_args: dict = docopt(__doc__, sys.argv[1:], version=__version__)

122

123 if global_args['--debug']:

124 coloredlogs.install(level=logging.DEBUG)

125

126 if global_args['--trace']:

127 coloredlogs.install(level=verboselogs.SPAM)

128 os.environ['DEBUG_HTTP'] = "true"

129

130 command_argv = [global_args['<command>']] + global_args['<args>']

131

132 logger.spam(f"Global arguments:\n {global_args}")

133 logger.spam(f"Command arguments:\n {command_argv}")

134

135 cmd = global_args['<command>']

136 if cmd == 'help':

137 _help(__doc__)

138 elif cmd == 'version':

139 _help(__version__)

140 elif cmd == 'case':

141 Case(global_args, command_argv)

142 elif cmd == 'list' or cmd == 'ls':

143 List(global_args, command_argv)

144 elif cmd == 'create':

145 Create(global_args, command_argv)

146 elif cmd == 'upload':

147 Upload(global_args, command_argv)

148 elif cmd == 'run':

149 Run(global_args, command_argv)

150 elif cmd == 'job':

151 Job(global_args, command_argv)

152 elif cmd == 'poll':

153 Poll(global_args, command_argv)

154 elif cmd == 'report':

155 Report(global_args, command_argv)

156 else:

157 _die(f"Command '{cmd}' is invalid. See 'pyriandx help'")

158

159

160class Command:

161

162 def __int__(self):

163 # sub-class should set these

164 self.case_id = None

165 self.client = None

166 self.resources = None

167

168 def get_case(self):

169 assert str(self.case_id).isnumeric(), _halt(f"Invalid Case ID: {self.case_id}", self.__doc__)

170 logger.info(f"Get a case with ID: {self.case_id}")

171 case = self.client.get_case_info(self.case_id)

172 assert case is not None and "id" in case, _halt(f"Case not found for ID: {self.case_id}", self.__doc__)

173 logger.debug(f"Found case with ID: {self.case_id}")

174 return case

175

176 def upload_case_files(self):

177 files = []

178 for r in self.resources:

179 if os.path.isdir(r):

180 case_files = [f for f in os.listdir(r) if os.path.isfile(os.path.join(r, f))]

181 for cf in case_files:

182 files.append(os.path.join(r, cf))

183 else:

184 files.append(r)

185

186 for f in files:

187 logger.info(f"Uploading case file: {f}")

188 self.client.upload_file(f, self.case_id)

189

190

191class Case(Command):

192 """Usage:

193 pyriandx case help

194 pyriandx case [options] <case-id>

195

196Description:

197 Get a case by given ID from PierianDx CGW. It returns in JSON

198 format. You can further process it e.g. pretty print by pipe

199 through with program such as jq.

200

201Example:

202 pyriandx case 69695

203 pyriandx case 69695 | jq

204 """

205

206 def __init__(self, global_args, command_argv):

207 args: dict = docopt(self.__doc__, argv=command_argv)

208 logger.spam(f"Case arguments:\n {args}")

209 assert args['case'] is True, _die("Command mismatch: Case")

210

211 if args['help']:

212 _help(self.__doc__)

213

214 self.client: Client = _build(global_args)

215 self.case_id = args['<case-id>']

216 self.case = self.get_case()

217 print(json.dumps(self.case)) # print here is intended i.e. pyriandx case 1234 | jq

218

219

220class List(Command):

221 """Usage:

222 pyriandx list help

223 pyriandx list [options] [<filters>...]

224

225Description:

226 List all cases by from PierianDx CGW. It returns in JSON format.

227 You can further process it e.g. pretty print by pipe through with

228 program such as jq. Optionally you can provide filters to limit

229 the return list.

230

231Allow filters:

232 id Case ID

233 accessionNumber Accession Number

234 panel The name of the case's panel

235 dateCreatedStart Inclusive start range for the date created

236 dateCreatedEnd Exclusive end range for the date created

237 dateSignedOutStart Inclusive start range for the date signed out

238 dateSignedOutEnd Exclusive end range for the date signed out

239

240Example:

241 pyriandx list

242 pyriandx list | jq

243 pyriandx list id=1234

244 pyriandx list accessionNumber=SBJ000123

245 pyriandx list dateSignedOutStart=2020-04-01

246 """

247

248 _F = ['id', 'accessionNumber', 'panel', 'dateCreatedStart',

249 'dateCreatedEnd', 'dateSignedOutStart', 'dateSignedOutEnd']

250

251 def __init__(self, global_args, command_argv):

252 args: dict = docopt(self.__doc__, argv=command_argv)

253 logger.spam(f"List arguments:\n {args}")

254 assert args['list'] is True, _die("Command mismatch: List")

255

256 if args['help']:

257 _help(self.__doc__)

258

259 self.client: Client = _build(global_args)

260 self.filters = args['<filters>']

261

262 logger.debug(f"Filters: {self.filters}")

263

264 params = {}

265 for ftr in self.filters:

266 assert '=' in ftr, _halt(f"Invalid filter supplied: {ftr}", self.__doc__)

267 fil = ftr.split('=')

268 assert fil[0] in self._F, _halt(f"Invalid filter supplied: {ftr}", self.__doc__)

269 params.update({fil[0]: fil[1]})

270

271 self.cases = self.client.list_cases(filters=params)

272 print(json.dumps(self.cases)) # print here is intended i.e. pyriandx list | jq

273

274

275class Upload(Command):

276 """Usage:

277 pyriandx upload help

278 pyriandx upload [options] <case-id> FILES...

279

280Description:

281 FILES... can be a directory that contains list of files that

282 stage to upload. Or, you can also provide individual file with

283 space separated for multiple of them.

284

285Example:

286 pyriandx upload 69695 path/to/SBJ00123/

287 pyriandx upload 69695 file1.vcf.gz file2.vcf.gz file3.cnv

288 """

289

290 def __init__(self, global_args, command_argv):

291 args: dict = docopt(self.__doc__, argv=command_argv)

292 logger.spam(f"Create arguments:\n {args}")

293 assert args['upload'] is True, _die("Command mismatch: Upload")

294

295 if args['help']:

296 _help(self.__doc__)

297

298 self.case_id = args['<case-id>']

299 self.resources = args['FILES']

300 self.client: Client = _build(global_args)

301

302 if self.get_case():

303 self.upload_case_files()

304

305

306class Create(Command):

307 """Usage:

308 pyriandx create help

309 pyriandx create [options] <json-file> [FILES...]

310

311Description:

312 Accession a new case from given input JSON file. Optionally,

313 FILES... can be a directory that contains list of files that

314 stage to upload. Or, you can also provide individual file with

315 space separated for multiple of them.

316

317Example:

318 pyriandx create my_case.json path/to/SBJ00123/

319 pyriandx create my_case.json file1.vcf.gz file2.vcf.gz file3.cnv

320 """

321

322 def __init__(self, global_args, command_argv):

323 args: dict = docopt(self.__doc__, argv=command_argv)

324 logger.spam(f"Create arguments:\n {args}")

325 assert args['create'] is True, _die("Command mismatch: Create")

326

327 if args['help']:

328 _help(self.__doc__)

329

330 self.input_file = args['<json-file>']

331 self.resources = args['FILES']

332

333 assert str(self.input_file).endswith('.json'), _halt(f"Case input file must be in JSON format", self.__doc__)

334 assert os.path.exists(self.input_file), _halt(f"No such file: {self.input_file}", self.__doc__)

335

336 self.client: Client = _build(global_args)

337

338 logger.info(f"Creating case from input file: {self.input_file}")

339 self.case_id = self.client.create_case(self.input_file)

340 logger.success(f"Created case with ID: {self.case_id}")

341

342 if self.resources:

343 self.upload_case_files()

344

345

346class Run(Command):

347 """Usage:

348 pyriandx run help

349 pyriandx run [options] <case-id>

350

351Description:

352 Create sequencer run for given Case ID. Note that each invocation

353 will create a sequencer run for given case. At the moment, it uses

354 internal `create_sequencer_run.json` template to create a sequencer

355 run. It returns Run ID. It will associate this Run ID with accession

356 number of given case. You typically need at least 1 sequencer run

357 after case has accessioned.

358

359Example:

360 pyriandx run 69695

361 > 1

362 pyriandx run 69695

363 > 2

364 pyriandx case 69695 | jq

365 pyriandx case 69695 | jq '.sequencerRuns[] | select(.runId == "1")'

366 """

367

368 def __init__(self, global_args, command_argv):

369 args: dict = docopt(self.__doc__, argv=command_argv)

370 logger.spam(f"Run arguments:\n {args}")

371 assert args['run'] is True, _die("Command mismatch: Run")

372

373 if args['help']:

374 _help(self.__doc__)

375

376 self.client: Client = _build(global_args)

377 self.case_id = args['<case-id>']

378

379 case = self.get_case()

380

381 if case:

382 self.accession_number = str(case['specimens'][0]['accessionNumber'])

383 next_run_id = 1 # start from 1

384

385 # check existing sequence run

386 if 'sequencerRuns' in case:

387 logger.info(f"Case ID {self.case_id} has existing sequencer runs:")

388 run_ids = []

389 for run in case['sequencerRuns']:

390 rid = run['runId']

391 logger.info(f"\tRun ID: {rid}, Date Created: {run['dateCreated']}")

392 if str(rid).isnumeric(): # ignore if not numeric

393 run_ids.append(rid)

394 if len(run_ids) > 0:

395 next_run_id = int(sorted(run_ids, reverse=True)[0]) + 1 # increase serial

396

397 logger.info(f"Creating sequencer run for case {self.case_id}")

398 id_ = self.client.create_sequencer_run(self.accession_number, next_run_id)

399 self.run_id = next_run_id

400 logger.success(f"Created sequencer run with ID: {self.run_id}")

401

402

403class Job(Command):

404 """Usage:

405 pyriandx job help

406 pyriandx job [options] <case-id> <run-id>

407

408Description:

409 Create informatics job for given Case ID and Run ID. At the moment,

410 it uses internal `create_job.json` template to create analysis job.

411 It returns Job ID. It will associate this informatics job with given

412 case. The analysis informatics job will kick off right away for the

413 given case and uploaded case files. Note that each invocation will

414 create a new informatics job for given case. It also implies that

415 you should create a case, a sequencer run and uploaded case files

416 before running informatics analysis job.

417

418Example:

419 pyriandx job 69695 1

420 > 19635

421 pyriandx job 69695 1

422 > 19636

423 pyriandx case 69695 | jq

424 pyriandx case 69695 | jq '.informaticsJobs[] | select(.id == "19635")'

425 """

426

427 def __init__(self, global_args, command_argv):

428 args: dict = docopt(self.__doc__, argv=command_argv)

429 logger.spam(f"Job arguments:\n {args}")

430 assert args['job'] is True, _die("Command mismatch: Job")

431

432 if args['help']:

433 _help(self.__doc__)

434

435 self.client: Client = _build(global_args)

436 self.case_id = args['<case-id>']

437 self.run_id = args['<run-id>']

438

439 case = self.get_case()

440

441 if 'caseFiles' not in case:

442 logger.warning(f"No case files found in your accessioned case. Very likely that informatics job may fail!")

443

444 assert 'sequencerRuns' in case, _halt(f"No sequencer run found in case {self.case_id}", self.__doc__)

445

446 found = False

447 for run in case['sequencerRuns']:

448 if run['runId'] == self.run_id:

449 found = True

450 continue

451 assert found is True, _halt(f"Sequencer run ID {self.run_id} is not found in case {self.case_id}", self.__doc__)

452

453 if case:

454 logger.info(f"Creating informatics job for case {self.case_id}")

455 self.job_id = self.client.create_job(case, self.run_id)

456 logger.success(f"Created informatics job with ID: {self.job_id}")

457

458

459class Poll(Command):

460 """Usage:

461 pyriandx poll help

462 pyriandx poll [options] <case-id> <job-id>

463

464Description:

465 Poll informatics job for given Case ID and Job ID. Maximum wait

466 time for polling job status is 30 minutes. It will timeout after

467 30 minutes. You can poll again. Alternatively, you can check the

468 informatics job status in PierianDx CGW dashboard. Or, get a case

469 and filter job ID on the return JSON using jq.

470

471 CAVEAT: Polling job status through API is not perfected yet. Please

472 do not rely on this feature for status check.

473

474Example:

475 pyriandx poll 69695 19635

476 pyriandx poll 69695 19636

477 pyriandx case 69695 | jq

478 pyriandx case 69695 | jq '.informaticsJobs[] | select(.id == "19635")'

479 pyriandx case 69695 | jq '.informaticsJobs[] | select(.id == "19635") | .status'

480 """

481

482 def __init__(self, global_args, command_argv):

483 args: dict = docopt(self.__doc__, argv=command_argv)

484 logger.spam(f"Poll arguments:\n {args}")

485 assert args['poll'] is True, _die("Command mismatch: Poll")

486

487 if args['help']:

488 _help(self.__doc__)

489

490 self.client: Client = _build(global_args)

491 self.case_id = args['<case-id>']

492 self.job_id = args['<job-id>']

493

494 case = self.get_case()

495 self.accession_number = str(case['specimens'][0]['accessionNumber'])

496 logger.info(f"Accession Number: {self.accession_number}")

497

498 self.complete = False

499 self.__start_poll()

500

501 def __start_poll(self):

502 status = self.client.get_job_status(self.case_id, self.job_id)

503 logger.info(f"Started polling job {self.job_id} status... (Ctrl+C to exit) ")

504

505 count = 0

506 while status != "complete" and status != "failed" and count < 60: # wait 30 minutes max

507 logger.info(f"Status is: {status}")

508 time.sleep(30) # Check API every 30 seconds

509 status = self.client.get_job_status(self.case_id, self.job_id)

510 count = count + 1

511

512 if count == 60:

513 logger.info("Job polling has reached timeout 30 minutes")

514 elif status == "complete":

515 logger.warning(f"Informatics job {self.job_id} for case {self.case_id} with accession number "

516 f"{self.accession_number} might have completed")

517 logger.warning(f"You should check in CGW dashboard to make sure it has completed successfully")

518 logger.warning(f"CLI API call does not able to differentiate `status` transition effectively at the moment")

519 self.complete = True

520 elif status == "failed":

521 logger.critical(f"Job did not complete, status was {status}")

522 logger.info(f"Please send support request to support@pieriandx.com with the following info:")

523 logger.info(f"Case ID: {self.case_id}")

524 logger.info(f"Job ID: {self.job_id}")

525 logger.info(f"Accession Number: {self.accession_number}")

526

527

528class Report(Command):

529 """Usage:

530 pyriandx report help

531 pyriandx report [options] <case-id>

532

533Description:

534 Get a report for given Case ID. It will download report in

535 PDF format and save it into ./output folder.

536

537 CAVEAT: Download report through API is not perfected yet.

538 Please do not rely on this feature.

539

540Example:

541 pyriandx report 69695

542 """

543

544 def __init__(self, global_args, command_argv):

545 args: dict = docopt(self.__doc__, argv=command_argv)

546 logger.spam(f"Report arguments:\n {args}")

547 assert args['report'] is True, _die("Command mismatch: Report")

548

549 if args['help']:

550 _help(self.__doc__)

551

552 self.client: Client = _build(global_args)

553 self.case_id = args['<case-id>']

554

555 logger.info(f"Finding report IDs for case: {str(self.case_id)}")

556 case_ = self.get_case()

557

558 if 'reports' not in case_:

559 logger.info(f"No reports available for case {self.case_id}. Try again later.")

560 else:

561 logger.info(f"Downloading report for case {self.case_id}")

562 self.client.get_report(case_, "output")

563 logger.success("Report download complete. Check in ./output folder")

564

565

566def main():

567 if len(sys.argv) == 1:

568 sys.argv.append('help')

569 python_version = ".".join(map(str, sys.version_info[:3]))

570 assert sys.version_info >= (3, 6), _die(f"This tool requires Python >=3.6. Found {python_version}")

571 try:

572 _dispatch()

573 except KeyboardInterrupt:

574 pass