Coverage for src/lcdoc/lp.py: 79.14%

1#!/usr/bin/env python

2"""

3Literate Programming / Language of Babel style

5Edit: We support this now not only from emacs but also from normal mkdocs markdown.

6See the documentation.

10Old emacs centric docs follow:

12Org Mode / Mkdocs Helper Script.

14Usage: See e.g. repos/lc/docs/.../getting_started.org

16Basics:

17 In you org file have this:

20#+BEGIN_SRC python :exports none :results silent :session pyroot

21from sys import modules as m; m.pop('lp') if 'lp' in m else 0; import lp

22#lp.alias('lcm', '$d_repos/lc/build/make')

23run = lp.p(lp.run, fmt='mk_cmd_out')

24root = lp.p(run, root=True)

25#+END_SRC

27which you execute at any change of the module (reloads)

29Set this:

31:PROPERTIES:

32:header-args:python: :session pyroot :exports results :results html

33:END:

35Then using is like

37#+BEGIN_SRC python

38root([

39 {

40 'cmd': 'm -l run lnr -d',

41 'expect': '\n/opt/repos/lc/node-red#',

42 'timeout': 15,

43 },

44 {

45 'cmd': "ax/start -p ''",

46 'expect': 'Started flows',

47 'timeout': 10,

48 'cmt': "-p '': no project"

49 },

50 ], new_session='root_nr')

51#+END_SRC

54"""

55import hashlib 2 ctxlppytest

56import html 2 ctxlppytest

57import json 2 ctxlppytest

58import os 2 ctxlppytest

59import string 2 ctxlppytest

60import subprocess as sp 2 ctxlppytest

61import sys 2 ctxlppytest

62import time 2 ctxlppytest

63from functools import partial as p 2 ctxlppytest

64from importlib import import_module 2 ctxlppytest

66import pycond 2 ctxlppytest

67from lcdoc import lprunner as mdr 2 ctxlppytest

68from lcdoc.mkdocs.tools import page_dir 2 ctxlppytest

69from lcdoc.tools import app, dirname, write_file 2 ctxlppytest

71# important (colorize)

72I = lambda s: s if not sys.stdout.isatty() else '\x1b[1;32m%s\x1b[0m' % s 72 ↛ exitline 72 didn't run the lambda on line 722 ctxlppytest

75is_lprunner = [0] 2 ctxlppytest

76env = os.environ 2 ctxlppytest

77wait = time.sleep 2 ctxlppytest

78now = time.time 2 ctxlppytest

79exists = os.path.exists 2 ctxlppytest

80user = env.get('USER', 'root') # in docker: Key error, else. 2 ctxlppytest

81env['PATH'] = 'bin:%s' % env.get('PATH', '') 2 ctxlppytest

82bash = lambda: os.system('bash') # debug 82 ↛ exitline 82 didn't run the lambda on line 822 ctxlppytest

85dbg = p(_, app.debug) 2 ctxlppytest

86nfo = p(_, app.info) 2 ctxlppytest

89# ---------------------------------------------------------------------------- Utilities

90class Raiser: 2 ctxlppytest

91 def raise_(ass): 2 ctxlppytest

92 msg = 'Assertion failed: Expected "%s" not found in result' % ass

93 # app.error(msg, asserts=ass, json={'result': res})

94 print('RAISING EXCEPTION: %s' % msg)

95 Raiser.die(msg)

97 def die(msg): 2 ctxlppytest

98 # this is monkey patched for gevent in lcdoc.__init__:

99 raise Exception(msg)

100

101

102def check_assert(ass, res): 2 ctxlppytest

108 is_pycond = True 3 ctxlp|features/lp/examples.mdlp|features/lp/parameters.mdlp|features/lp/parameters.md

109 elif isinstance(ass, list) and ass[1] in {'and', 'or'}: 109 ↛ 110line 109 didn't jump to line 110, because the condition on line 109 was never true12 ctxlp|features/lp/examples.mdlp|features/lp/examples.mdlp|features/lp/examples.mdlp|features/lp/parameters.mdlp|features/lp/parameters.mdlp|features/lp/parameters.mdlp|features/lp/parameters.mdlp|features/lp/parameters.mdlp|features/lp/syntax.mdlp|features/lp/syntax.mdlp|features/lp/syntax.mdlp|features/lp/tips.md

110 is_pycond = True

112

113 def f(k, v, state, **kw): 3 ctxlp|features/lp/examples.mdlp|features/lp/parameters.mdlp|features/lp/parameters.md

114 return (k in state['res'], v) 3 ctxlp|features/lp/examples.mdlp|features/lp/parameters.mdlp|features/lp/parameters.md

115

116 r = pycond.pycond(ass, f) 3 ctxlp|features/lp/examples.mdlp|features/lp/parameters.mdlp|features/lp/parameters.md

117 a = r(state={'res': s}) 3 ctxlp|features/lp/examples.mdlp|features/lp/parameters.mdlp|features/lp/parameters.md

118 if not a: 118 ↛ 119line 118 didn't jump to line 119, because the condition on line 118 was never true3 ctxlp|features/lp/examples.mdlp|features/lp/parameters.mdlp|features/lp/parameters.md

119 Raiser.raise_(ass)

120 return 3 ctxlp|features/lp/examples.mdlp|features/lp/parameters.mdlp|features/lp/parameters.md

121

124

126

128 Raiser.raise_(ass)

131# aliases = {}

135def repl_dollar_var_with_env_val(s, die_on_fail=True): 2 ctxlppytest

138 for k in env:

139 if k in s:

140 s = s.replace('$' + k, env[k])

141 if '$' in s:

142 dbg('Not defined in environ: $%s' % s)

143 return s

144

145

146def spresc(cmd): 2 ctxlppytest

147 """subprocess run - with escape sequences escaped (for tmux)"""

149

150

151def sprun(*a, no_fail=False, report=False, **kw): 2 ctxlppytest

152 """Running a command as bash subprocess

153 W/o executable we crash on ubuntu's crazy dash, which cannot even echo -e

154 """

156 if not kw and len(a) == 1:

157 report('', a[0])

158 else:

159 report('', a, kw) # for the user (also in view messages)

162 if isinstance(a[0], str) and not str(a[0]).startswith('tmux'):

163 show_res = True

164 mdr.confirm('Run %s' % ' '.join(a))

165

168 # a line within bash lp ... has e.g. # pdb -> stop before executing command

169 # so that user can try:

171 print('"pdb" was found, interrupting before calling:')

172 print(a[0])

173 print('type bash()` to get a shell.')

174 breakpoint() # FIXME BREAKPOINT

177 print('Result:')

178 print(res.decode('utf-8'))

180 except Exception as ex:

181 if not no_fail:

182 raise

183 app.error('Failure', args=a, kw=kw, exc=ex)

184 return err(str(ex))

185

186

187# ----------------------------------------------------------------------- Plugins(modes)

188

189our_plugs = 'lcdoc.mkdocs.lp.plugs' 2 ctxlppytest

190

191

192class plugs: 2 ctxlppytest

193 pass 2 ctxlppytest

194

195

196def get_or_import_plug(mode): 2 ctxlppytest

202 if not hasattr(p, 'run'):

203 raise ModuleNotFoundError('')

207 except ModuleNotFoundError as ex:

208 msg = ' - If this is a custom module add its directory to your PYTHONPATH'

209 raise type(ex)(str(ex) + msg)

212

213

214# ------------------------------------------------------------------------------ Formats

215# seems we don't need the stupid dot anymore for the javascript xterm part (?)

216# mk_cmd_out = '''

217

218# === "Cmd"

219

220# ```console

221# %(cmds)s

222# ```

224# .

226# === "Output"

228# %(ress)s

230# '''

232mk_console = '''

233```%(lang)s

234%(cmd)s

235%(res)s

236```

237'''

240mk_cmd_out = '''

242=== "Cmd"

244 ```console

245 %(cmds)s

246 ```

248=== "Output"

250 %(ress)s

252'''

255class mk_cmd_out_fetch: 2 ctxlppytest

256 org = '''

257

258

259#+begin_tab:Command

260

261#+BEGIN_SRC console :eval no

263%(cmds)s

265#+END_SRC

267#+end_tab:Command

270#+begin_tab:Output

271

272%(ress)s

273

274#+end_tab:Output

275

276'''

277 mkdocs = mk_cmd_out 2 ctxlppytest

278

279

280xt_flat = '''

281<xterm %(root)s/>

282

283 %(ress)s

284

285'''

286

287xt_flat_fetch = '''

288 <xterm />

289

290 remote_content

291

292 ![](%(fn_frm)s)

293'''

294

295

296# xt_flat_fetch_test = """

297##+begin_tab:foo

298

299# <xtf %(root)s xtfrm=file:%(fn_frm)s></xtf>

300

301##+end_tab:foo

302# """

303

304# this not yet sane:

305##+begin_tab:foo

306

307# <xterm></xterm>

308

309# ansiremotecontent

310

311# [ ](file:./media/info_flow_adm.ansi)

312

313##+end_tab:foo

314

315

316dflt_fmt = 'mk_cmd_out' 2 ctxlppytest

317

318

319class T: 2 ctxlppytest

320 def xt_flat(resl, **kw): 2 ctxlppytest

327 res = cmdstr + '\n' + res

332 fn_frm = kw['fn_frm'] = file_.write_fetchable(ress, **kw) 1 ctxlp|features/lp/parameters.md

333 t = xt_flat_fetch 1 ctxlp|features/lp/parameters.md

334 ress = t % locals() 1 ctxlp|features/lp/parameters.md

335 else:

339

340 def mk_cmd_out(res, **kw): 2 ctxlppytest

342

344 """

345 We don't do it since console rendering requires $ or # only for syn hilite

346 """

353 if len(c) + len(cmt) > 80:

354 c = '%s # %s:\n%s' % (p, cmt, c)

355 else:

356 c = '%s # %s' % (c, cmt)

358

362 return ress 1 ctxlp|features/lp/parameters.md

366 t = getattr(mk_cmd_out_fetch, kw['fetched_block_fmt']) 1 ctxlp|features/lp/parameters.md

367 r = t % locals() 1 ctxlp|features/lp/parameters.md

368 else:

369 # indent one in:

374

375 def mk_console(res, root=False, **kw): 2 ctxlppytest

386 # is the command part of the res? then skip print:

388 cmd = 'SKIP-PRINT-OUT' 4 ctxlp|features/lp/parameters.mdlp|features/lp/parameters.mdlp|features/lp/plugs/make_file/index.mdlp|features/lp/syntax.md

390

394

395

396T.xtf = T.xt_flat 2 ctxlppytest

397

398

399def get_cmd(res): 2 ctxlppytest

402

403

404# def alias(k, v):

405# aliases[k + ' '] = v + ' '

406

407

408def prepare_and_fmt(res, orig_cmd, **kw): 2 ctxlppytest

409 # allow plugs to do their formatting:

418 c = res[i] = {}

420 app.debug('Command missing', res=o, cmd=orig_cmd) 1 ctxlp|features/lp/python/_tech.md

421 c['cmd'] = orig_cmd 1 ctxlp|features/lp/python/_tech.md

423 app.error('Result missing', res=o, cmd=orig_cmd)

424 c['res'] = 'Not available (%s)' % orig_cmd

428 else:

429 c['res'] = json.dumps(r, indent=4)

431 c['cmd'] = '' 1 ctxlp|features/lp/parameters.md

432

436 raise Exception(

437 'Format not found (%s) - have: %s'

438 % (f, [k for k in dir(T) if not k[0] == '_'])

439 )

441

442

443letters = string.ascii_letters + string.digits + '_' 2 ctxlppytest

444

445

446# ----------------------------------------------------------------------- header parsing

447def cast(v): 2 ctxlppytest

449 return json.loads(v) 1 ctxlp|features/lp/examples.md

457

458

459apos = ["'", '"'] 2 ctxlppytest

460neutrl_chars = [[' ', '\x01'], [',', '\x02'], ['=', '\x04']] 2 ctxlppytest

461

462

463def neutralize(s, mode=None): 2 ctxlppytest

469

470

471def parse_kw_str(kws, header_kws=None, try_json=True): 2 ctxlppytest

472 """for kw via cli"""

474 return {} 9 ctxlp|about/changelog.mdlp|about/credits.mdlp|features/lp/examples.mdlp|features/lp/parameters.mdlp|features/lp/plugs/chart/index.mdlp|features/lp/plugs/column/index.mdlp|features/lp/plugs/flowchart/index.mdlp|features/lp/plugs/kroki/index.mdlp|features/lp/sessions.md

477 if kws and kws[0] in ('{', '['):

478 try:

479 return json.loads(kws)

480 except:

481 pass

482 # neutralize all within apos:

491 msg = 'Unparsable apostrophes: %s - %s'

492 raise Exception(msg % (kws, apo))

495

496 # quoted values possible:

498 raise Exception('No comma allowed') 1 ctxlp|features/lp/examples.md

502 {

503 p[0]: cast(neutralize(p[1], 'rev'))

504 for p in [(k if '=' in k else k + '=true').split('=') for k in parts]

505 }

506 )

507 # breakpoint() # FIXME BREAKPOINT

508 # if '"' in kws or "'" in kws:

509 # r = []

510 # while kw:

511 # k, v = kw.pop(0)

512 # for apo in apos:

513 # if v[0] == apo:

514 # v = v[1:]

515 # if v.endswith(apo):

516 # v = v[:-1]

517 # break

518 # while kw:

519 # nk, _ = kw.pop(0)

520 # if nk.endswith(apo):

521 # v += ' ' + nk[:-1]

522 # break

523 # else:

524 # v += ' ' + nk

525 # break

526 # r.append([k, v])

527 # kw = r

528 # kw = {k: cast(v) for k, v in kw}

529

532

533

534def parse_header_args(header, **ctx): 2 ctxlppytest

535 """Parsing either python or easy format

536 CAUTION: This is used

537 - caused here, to parse inline headers ( # lp: ....)

538 - in the plugin code as well, to parse markdown headers

539 """

540 # ctx['dir_repo'] = ctx['fn_lp'].split('/docs/', 1)[0]

542

544 # if header.rstrip().endswith('"'):

545 # raise Exception("Not tried (apostrophe at end)")

547 except Exception as ex: 2 ctxlp|features/lp/examples.mdlp|features/lp/examples.md

548 ex1 = ex 2 ctxlp|features/lp/examples.mdlp|features/lp/examples.md

549 # evaling now.

550 # still - we supply only a minimum eval ctx and prevent

551 # imports. But still this won't be totally safe.

552 # BUT: Hey - we are about to run code *anyway*, this is LP in the end!

553 if not 'import' in header: 553 ↛ 559line 553 didn't jump to line 559, because the condition on line 553 was never false2 ctxlp|features/lp/examples.mdlp|features/lp/examples.md

554 try: 2 ctxlp|features/lp/examples.mdlp|features/lp/examples.md

555 m = eval('get_args(%s)' % header, ctx, {}) 2 ctxlp|features/lp/examples.mdlp|features/lp/examples.md

556 return 0, (m['args'], m['kw']) 2 ctxlp|features/lp/examples.mdlp|features/lp/examples.md

557 except Exception as ex:

558 ex2 = ex

559 return 1, (ex2, ex1)

560

561

562# ----------------------------------------------------------------------------- sessions

563

564

565def pb(s): 2 ctxlppytest

566 try:

567 s = s.decode('utf-8')

568 except Exception as ex:

569 pass

570 dbg(s)

571

572

573def get_args(*a, **kw): 2 ctxlppytest

574 return {'args': a, 'kw': kw} 2 ctxlp|features/lp/examples.mdlp|features/lp/examples.md

575

576

577def root(cmd, **kw): 2 ctxlppytest

578 return run(cmd, root=True, **kw)

579

580

581class file_: 2 ctxlppytest

582 def write_fetchable(cont, fetch, **kw): 2 ctxlppytest

583 """write .ansi XTF files"""

584 d, fn = kw['fn_doc'].rsplit('/', 1) 1 ctxlp|features/lp/parameters.md

585 lnk = '/media/%s_%s.ansi' % (fn, fetch) 1 ctxlp|features/lp/parameters.md

586 fn = d + lnk 1 ctxlp|features/lp/parameters.md

587 if not exists(d + '/media'): 587 ↛ 589line 587 didn't jump to line 589, because the condition on line 587 was never false1 ctxlp|features/lp/parameters.md

588 os.makedirs(d + '/media') 1 ctxlp|features/lp/parameters.md

589 s = rpl(cont, kw) 1 ctxlp|features/lp/parameters.md

590 write_file(fn, s, only_on_change=True) 1 ctxlp|features/lp/parameters.md

591 return '.' + lnk 1 ctxlp|features/lp/parameters.md

592

593

594def rpl(s, kw): 2 ctxlppytest

595 'global replacement'

598 if not isinstance(rpl[0], (list, tuple)):

599 rpl = [rpl]

600 for r in rpl:

601 s = s.replace(r[0], r[1])

603

604

605def repl_dollar_var_with_env_vals(kw, *keys): 2 ctxlppytest

610

611

612def multi_line_to_list(cmd): 2 ctxlppytest

613 """

614

615 ```bash lp session=DO asserts=loadbalancer.tf

616 ip () { echo 1.2.3.4; }

617 cat << FOO > loadbalancer.tf

618 > resource "digitalocean_loadbalancer" "www-lb" {

619 > ipv4 = $(ip)

620 > (...)

621 >}

622 >FOO

623 ls -l

624 ```

625

626 """

628 return cmd 2 ctxlp|features/lp/parameters.mdlp|features/lp/parameters.md

630 # when some lines start with ' ' we send the whole cmd as one string:

632 # echo 'foo

633 # bar' > baz

634 return [{'cmd': cmd}] 1 ctxlp|features/lp/examples.md

640 l = lines.pop(0) 3 ctxlp|features/lp/examples.mdlp|features/lp/syntax.mdlp|features/lp/tips.md

641 r[-1] += '\n' + l[2:] 3 ctxlp|features/lp/examples.mdlp|features/lp/syntax.mdlp|features/lp/tips.md

644

645

646def check_inline_lp(cmd): 2 ctxlppytest

648 breakpoint() # FIXME BREAKPOINT

649 return

655 msg = 'Inline lp construct wrong: %s. Valid e.g.: "ls -lta /etc # lp: '

656 msg += 'expect=hosts timeout=10". Got: %s %s'

657 raise Exception(msg % (cmd, res[0], res[1]))

660

661

662def os_system_if_param_present(kw, if_present): 2 ctxlppytest

663 """utility for a frequent use case"""

665 return

668 if os.system(cmd):

669 raise Exception('%s run failed: %s. kw: %s' % (if_present, cmd, str(kw)))

670

671

672def err(msg, **kw): 2 ctxlppytest

673 app.error(msg, **kw)

674 return {'res': {'lp err': msg, 'kw': kw}}

675

676

677class SessionNS: 2 ctxlppytest

678 """Namespace for methods related to session parameters

679

680 Including methods when there is no session support

681 (but different handling when there is)

682 """

683

684 @classmethod 2 ctxlppytest

685 def init(cls, cmd, kw):

689

690 @classmethod 2 ctxlppytest

691 def handle_new_session_param(cls, kw):

696 raise Exception(msg)

698 ns = kw.pop('session')

699 if not ns:

700 raise Exception(msg)

704

705 @classmethod 2 ctxlppytest

706 def delete(cls, session_name, kw):

707 raise Exception('No session support', mode=kw.get('mode'))

708

709 @classmethod 2 ctxlppytest

710 def pre(cls, session_name, kw):

711 # pre, if not overwritten here should just os.system it:

713

714 @classmethod 2 ctxlppytest

715 def post(cls, session_name, kw):

717

718

719def eval_lp(cmd, kw): 2 ctxlppytest

720 # deprecated alias, not any more docued, did not work for py style args:

727 res = cmd = res.get('result') or res.get('res') 2 ctxlp|features/lp/plugs/chart/index.mdlp|features/lp/plugs/flowchart/index.md

728 if isinstance(res, dict) and res.get('header'): 728 ↛ 729line 728 didn't jump to line 729, because the condition on line 728 was never true2 ctxlp|features/lp/plugs/chart/index.mdlp|features/lp/plugs/flowchart/index.md