Coverage for src/auto_intersphinx/update_catalog.py: 99%

1# SPDX-FileCopyrightText: Copyright © 2022 Idiap Research Institute <contact@idiap.ch> 

2# 

3# SPDX-License-Identifier: BSD-3-Clause 

4 

5from __future__ import annotations # not required for Python >= 3.10 

6 

7import argparse 

8import importlib.abc 

9import os 

10import pathlib 

11import re 

12import sys 

13import textwrap 

14 

15import requests 

16 

17from sphinx.util import logging 

18 

19from . import oneliner 

20from .catalog import BUILTIN_CATALOG, Catalog 

21 

22logger = logging.getLogger(__name__) 

23 

24 

25def _parse_requirements(contents: str) -> list[str]: 

26 """Parses a pip-requirements file and extracts package lists.""" 

27 lines = contents.split() 

28 lines = [k.strip() for k in lines if not k.strip().startswith("#")] 

29 split_re = re.compile(r"[=\s]+") 

30 return [split_re.split(k)[0] for k in lines] 

31 

32 

33def setup_verbosity(verbose: int) -> None: 

34 """Sets up logger verbosity.""" 

35 import logging as builtin_logging 

36 

37 package_logger = builtin_logging.getLogger("sphinx." + __package__) 

38 

39 handler = builtin_logging.StreamHandler(sys.stdout) 

40 formatter = builtin_logging.Formatter("[%(levelname)s] %(message)s") 

41 handler.setFormatter(formatter) 

42 handler.setLevel(builtin_logging.DEBUG) 

43 

44 package_logger.addHandler(handler) 

45 if verbose == 0: 

46 package_logger.setLevel(builtin_logging.ERROR) 

47 elif verbose == 1: 

48 package_logger.setLevel(builtin_logging.WARNING) 

49 elif verbose == 2: 

50 package_logger.setLevel(builtin_logging.INFO) 

51 else: 

52 package_logger.setLevel(builtin_logging.DEBUG) 

53 

54 

55def _main(args) -> None: 

56 """Main function, that actually executes the update-catalog command.""" 

57 setup_verbosity(args.verbose) 

58 

59 catalog = Catalog() 

60 

61 if isinstance(args.catalog, str): 

62 catalog_path = pathlib.Path(args.catalog) 

63 if catalog_path.exists(): 

64 catalog.load(catalog_path) 

65 else: 

66 logger.info( 

67 f"Input catalog file `{str(args.catalog)}' does not " 

68 f"exist. Skipping..." 

69 ) 

70 elif isinstance(args.catalog, importlib.abc.Traversable): 

71 catalog.loads(args.catalog.read_text()) 

72 

73 if args.self: 

74 catalog.self_update() 

75 

76 package_list = [] 

77 for pkg in args.packages: 

78 if pkg.startswith("http"): 

79 logger.info(f"Retrieving package list from `{pkg}'...") 

80 r = requests.get(pkg) 

81 if r.ok: 

82 package_list += _parse_requirements(r.text) 

83 else: 

84 logger.error(f"Could not retrieve `{pkg}'") 

85 sys.exit(1) 

86 

87 elif os.path.exists(pkg): 

88 with open(pkg) as f: 

89 package_list += _parse_requirements(f.read()) 

90 

91 else: 

92 package_list.append(pkg) 

93 

94 if package_list: 

95 catalog.update_versions( 

96 pkgs=package_list, 

97 pypi_max_entries=args.pypi_max_entries, 

98 keep_going=args.keep_going, 

99 ) 

100 

101 if args.output: 

102 catalog.dump(args.output) 

103 

104 if (args.self or package_list) and args.output is None: 

105 # an update was run, dump results 

106 print(catalog.dumps()) 

107 

108 

109def add_parser(subparsers) -> argparse.ArgumentParser: 

110 """Just sets up the parser for this CLI.""" 

111 prog = "auto-intersphinx update-catalog" 

112 

113 parser = subparsers.add_parser( 

114 prog.split()[1], 

115 help="Updates catalog of intersphinx cross-references", 

116 description="Updates catalog of intersphinx cross-references", 

117 formatter_class=argparse.RawDescriptionHelpFormatter, 

118 epilog=textwrap.dedent( 

119 f""" 

120 examples: 

121 

122 1. Updates numpy and scipy from the internal catalog, dumps new 

123 catalog to stdout 

124 

125 .. code:: sh 

126 

127 {prog} numpy scipy 

128 

129 2. Self-update internal catalog: 

130 

131 .. code:: sh 

132 

133 {prog} --self 

134 

135 3. Refresh internal catalog from a remote pip-constraints.txt file: 

136 

137 .. code:: sh 

138 

139 {prog} https://gitlab.idiap.ch/software/idiap-citools/-/raw/main/src/citools/data/pip-constraints.txt 

140 

141 4. Run local tests without modifying the package catalog: 

142 

143 .. code:: sh 

144 

145 {prog} --output=testout.json 

146 """ 

147 ), 

148 ) 

149 

150 parser.add_argument( 

151 "-v", 

152 "--verbose", 

153 action="count", 

154 default=0, 

155 help="Can be set multiple times to increase console verbosity", 

156 ) 

157 

158 parser.add_argument( 

159 "--self", 

160 action="store_true", 

161 default=False, 

162 help="If set, then self-updates catalog entries", 

163 ) 

164 

165 parser.add_argument( 

166 "-o", 

167 "--output", 

168 type=pathlib.Path, 

169 help="If set, then dump the existing catalog to an output file", 

170 ) 

171 

172 parser.add_argument( 

173 "-c", 

174 "--catalog", 

175 default=BUILTIN_CATALOG, 

176 help="Location of the catalog to update [default: %(default)s]", 

177 ) 

178 

179 parser.add_argument( 

180 "-M", 

181 "--pypi-max-entries", 

182 default=0, 

183 type=int, 

184 help=oneliner( 

185 """ 

186 The maximum number of entries to lookup in PyPI. A value of zero 

187 will download only the main package information and will hit PyPI 

188 only once. A value bigger than zero will download at most the 

189 information from the last ``max_entries`` releases. Finally, a 

190 negative value will imply the download of all available releases. 

191 """ 

192 ), 

193 ) 

194 

195 parser.add_argument( 

196 "-k", 

197 "--keep-going", 

198 action="store_true", 

199 default=False, 

200 help=oneliner( 

201 """ 

202 If set, then do not stop at first found reference (such as 

203 auto-intersphinx would do), but rather keep searching for all 

204 references. 

205 """ 

206 ), 

207 ) 

208 

209 parser.add_argument( 

210 "packages", 

211 nargs="*", 

212 default=[], 

213 help=oneliner( 

214 """ 

215 Location of the reference package list to load to populate catalog. 

216 If not specified, then does not update anything (unless --self is 

217 set, of course). This argument may take multiple inputs. Each 

218 input may be a filename, which contains package lists, a remote 

219 http/https file, or a package name. 

220 """ 

221 ), 

222 ) 

223 

224 parser.set_defaults(func=_main) 

225 

226 return parser