diff --git a/tensorflow/tools/api/tests/api_compatibility_test.py b/tensorflow/tools/api/tests/api_compatibility_test.py index acd119c7305e8f..1ffa8fc26c0c69 100644 --- a/tensorflow/tools/api/tests/api_compatibility_test.py +++ b/tensorflow/tools/api/tests/api_compatibility_test.py @@ -198,7 +198,7 @@ def testAPIBackwardsCompatibility(self): visitor = python_object_to_proto_visitor.PythonObjectToProtoVisitor() public_api_visitor = public_api.PublicAPIVisitor(visitor) - public_api_visitor.do_not_descend_map[''].append('contrib') + public_api_visitor.do_not_descend_map['tf'].append('contrib') traverse.traverse(tf, public_api_visitor) proto_dict = visitor.GetProtos() diff --git a/tensorflow/tools/common/public_api.py b/tensorflow/tools/common/public_api.py index cab3b2ff6a0d39..e0acead9195933 100644 --- a/tensorflow/tools/common/public_api.py +++ b/tensorflow/tools/common/public_api.py @@ -36,12 +36,20 @@ def __init__(self, visitor): visitor: A visitor to call for the public API. """ self._visitor = visitor + self._root_name = 'tf' + + # Modules/classes we want to suppress entirely. + self._private_map = { + # Some implementations have this internal module that we shouldn't + # expose. + 'tf.flags': ['cpp_flags'], + } # Modules/classes we do not want to descend into if we hit them. Usually, # system modules exposed through platforms for compatibility reasons. # Each entry maps a module path to a name to ignore in traversal. self._do_not_descend_map = { - '': [ + 'tf': [ 'core', 'examples', 'flags', # Don't add flags @@ -56,17 +64,25 @@ def __init__(self, visitor): 'tensorboard', ], - # Some implementations have this internal module that we shouldn't - # expose. - 'flags': ['cpp_flags'], - ## Everything below here is legitimate. # It'll stay, but it's not officially part of the API. - 'app': ['flags'], + 'tf.app': ['flags'], # Imported for compatibility between py2/3. - 'test': ['mock'], + 'tf.test': ['mock'], } + @property + def private_map(self): + """A map from parents to symbols that should not be included at all. + + This map can be edited, but it should not be edited once traversal has + begun. + + Returns: + The map marking symbols to not include. + """ + return self._private_map + @property def do_not_descend_map(self): """A map from parents to symbols that should not be descended into. @@ -79,11 +95,17 @@ def do_not_descend_map(self): """ return self._do_not_descend_map - def _isprivate(self, name): + def set_root_name(self, root_name): + """Override the default root name of 'tf'.""" + self._root_name = root_name + + def _is_private(self, path, name): """Return whether a name is private.""" # TODO(wicke): Find out what names to exclude. - return (name.startswith('_') and not re.match('__.*__$', name) or - name in ['__base__', '__class__']) + return ((path in self._private_map and + name in self._private_map[path]) or + (name.startswith('_') and not re.match('__.*__$', name) or + name in ['__base__', '__class__'])) def _do_not_descend(self, path, name): """Safely queries if a specific fully qualified name should be excluded.""" @@ -95,17 +117,21 @@ def __call__(self, path, parent, children): # Avoid long waits in cases of pretty unambiguous failure. if tf_inspect.ismodule(parent) and len(path.split('.')) > 10: - raise RuntimeError('Modules nested too deep:\n%s\n\nThis is likely a ' - 'problem with an accidental public import.' % path) + raise RuntimeError('Modules nested too deep:\n%s.%s\n\nThis is likely a ' + 'problem with an accidental public import.' % + (self._root_name, path)) + + # Includes self._root_name + full_path = '.'.join([self._root_name, path]) if path else self._root_name # Remove things that are not visible. for name, child in list(children): - if self._isprivate(name): + if self._is_private(full_path, name): children.remove((name, child)) self._visitor(path, parent, children) # Remove things that are visible, but which should not be descended into. for name, child in list(children): - if self._do_not_descend(path, name): + if self._do_not_descend(full_path, name): children.remove((name, child)) diff --git a/tensorflow/tools/docs/doc_generator_visitor.py b/tensorflow/tools/docs/doc_generator_visitor.py index 8f7b91fa752f9e..259a4694fdcc00 100644 --- a/tensorflow/tools/docs/doc_generator_visitor.py +++ b/tensorflow/tools/docs/doc_generator_visitor.py @@ -30,7 +30,7 @@ def __init__(self, root_name=''): """Make a visitor. As this visitor is starting its traversal at a module or class, it will not - be old the name of that object during traversal. `root_name` is the name it + be told the name of that object during traversal. `root_name` is the name it should use for that object, effectively prefixing all names with "root_name.". diff --git a/tensorflow/tools/docs/generate_1_0.py b/tensorflow/tools/docs/generate_1_0.py index ddafcebd118035..cdc03fdcacf44f 100644 --- a/tensorflow/tools/docs/generate_1_0.py +++ b/tensorflow/tools/docs/generate_1_0.py @@ -47,8 +47,8 @@ doc_generator.set_py_modules([('tf', tf), ('tfdbg', tf_debug)]) doc_generator.set_do_not_descend_map({ - '': ['cli', 'lib', 'wrappers'], - 'contrib': [ + 'tf': ['cli', 'lib', 'wrappers'], + 'tf.contrib': [ 'compiler', 'factorization', 'grid_rnn', @@ -65,18 +65,18 @@ 'training', 'tfprof', ], - 'contrib.bayesflow': [ + 'tf.contrib.bayesflow': [ 'entropy', 'monte_carlo', 'special_math', 'stochastic_gradient_estimators', 'stochastic_graph', 'stochastic_tensor', 'stochastic_variables', 'variational_inference' ], - 'contrib.distributions': ['bijector'], - 'contrib.ffmpeg': ['ffmpeg_ops'], - 'contrib.graph_editor': [ + 'tf.contrib.distributions': ['bijector'], + 'tf.contrib.ffmpeg': ['ffmpeg_ops'], + 'tf.contrib.graph_editor': [ 'edit', 'match', 'reroute', 'subgraph', 'transform', 'select', 'util' ], - 'contrib.layers': ['feature_column', 'summaries'], - 'contrib.learn': [ + 'tf.contrib.layers': ['feature_column', 'summaries'], + 'tf.contrib.learn': [ 'datasets', 'head', 'graph_actions', @@ -87,7 +87,7 @@ 'preprocessing', 'utils', ], - 'contrib.util': ['loader'], + 'tf.contrib.util': ['loader'], }) sys.exit(doc_generator.build(flags)) diff --git a/tensorflow/tools/docs/generate_lib.py b/tensorflow/tools/docs/generate_lib.py index 3494c7f8c5a9d7..99872e1d8446ab 100644 --- a/tensorflow/tools/docs/generate_lib.py +++ b/tensorflow/tools/docs/generate_lib.py @@ -180,11 +180,16 @@ def add_dict_to_dict(add_from, add_to): # Exclude some libaries in contrib from the documentation altogether. +def _get_default_private_map(): + return {} + + +# Exclude members of some libaries. def _get_default_do_not_descend_map(): - # TODO(wicke): Shrink this list. + # TODO(wicke): Shrink this list once the modules get sealed. return { - '': ['cli', 'lib', 'wrappers'], - 'contrib': [ + 'tf': ['cli', 'lib', 'wrappers'], + 'tf.contrib': [ 'compiler', 'factorization', 'grid_rnn', @@ -200,17 +205,17 @@ def _get_default_do_not_descend_map(): 'testing', 'tfprof', ], - 'contrib.bayesflow': [ + 'tf.contrib.bayesflow': [ 'special_math', 'stochastic_gradient_estimators', 'stochastic_variables' ], - 'contrib.ffmpeg': ['ffmpeg_ops'], - 'contrib.graph_editor': [ + 'tf.contrib.ffmpeg': ['ffmpeg_ops'], + 'tf.contrib.graph_editor': [ 'edit', 'match', 'reroute', 'subgraph', 'transform', 'select', 'util' ], - 'contrib.keras': ['api', 'python'], - 'contrib.layers': ['feature_column', 'summaries'], - 'contrib.learn': [ + 'tf.contrib.keras': ['api', 'python'], + 'tf.contrib.layers': ['feature_column', 'summaries'], + 'tf.contrib.learn': [ 'datasets', 'head', 'graph_actions', @@ -221,15 +226,17 @@ def _get_default_do_not_descend_map(): 'preprocessing', 'utils', ], - 'contrib.util': ['loader'], + 'tf.contrib.util': ['loader'], } -def extract(py_modules, do_not_descend_map): +def extract(py_modules, private_map, do_not_descend_map): """Extract docs from tf namespace and write them to disk.""" # Traverse the first module. visitor = doc_generator_visitor.DocGeneratorVisitor(py_modules[0][0]) api_visitor = public_api.PublicAPIVisitor(visitor) + api_visitor.set_root_name(py_modules[0][0]) + add_dict_to_dict(private_map, api_visitor.private_map) add_dict_to_dict(do_not_descend_map, api_visitor.do_not_descend_map) traverse.traverse(py_modules[0][1], api_visitor) @@ -237,6 +244,7 @@ def extract(py_modules, do_not_descend_map): # Traverse all py_modules after the first: for module_name, module in py_modules[1:]: visitor.set_root_name(module_name) + api_visitor.set_root_name(module_name) traverse.traverse(module, api_visitor) return visitor @@ -409,6 +417,7 @@ class DocGenerator(object): def __init__(self): self.argument_parser = argparse.ArgumentParser() self._py_modules = None + self._private_map = _get_default_private_map() self._do_not_descend_map = _get_default_do_not_descend_map() self.yaml_toc = True @@ -439,9 +448,15 @@ def parse_known_args(self): flags, _ = self.argument_parser.parse_known_args() return flags + def add_to_private_map(self, d): + add_dict_to_dict(d, self._private_map) + def add_to_do_not_descend_map(self, d): add_dict_to_dict(d, self._do_not_descend_map) + def set_private_map(self, d): + self._private_map = d + def set_do_not_descend_map(self, d): self._do_not_descend_map = d @@ -471,7 +486,8 @@ def make_parser_config(self, visitor, reference_resolver, guide_index, base_dir=base_dir) def run_extraction(self): - return extract(self._py_modules, self._do_not_descend_map) + return extract( + self._py_modules, self._private_map, self._do_not_descend_map) def build(self, flags): """Actually build the docs.""" diff --git a/tensorflow/tools/docs/generate_lib_test.py b/tensorflow/tools/docs/generate_lib_test.py index 49f7ff597ed75e..6e5deb6a36ed7d 100644 --- a/tensorflow/tools/docs/generate_lib_test.py +++ b/tensorflow/tools/docs/generate_lib_test.py @@ -58,8 +58,9 @@ def test_extraction(self): py_modules = [('tf', tf), ('tfdbg', tf_debug)] try: - generate_lib.extract( - py_modules, generate_lib._get_default_do_not_descend_map()) + generate_lib.extract(py_modules, + generate_lib._get_default_private_map(), + generate_lib._get_default_do_not_descend_map()) except RuntimeError: print('*****************************************************************') print('If this test fails, you have most likely introduced an unsealed')