Make API do not descend map a bit more precise by including

the root module name. Add ability to mark symbols as private.

PiperOrigin-RevId: 158563334

tensorflow/tools/api/tests/api_compatibility_test.py

@@ -198,7 +198,7 @@ def testAPIBackwardsCompatibility(self):
  visitor = python_object_to_proto_visitor.PythonObjectToProtoVisitor()
 
  public_api_visitor = public_api.PublicAPIVisitor(visitor)
- public_api_visitor.do_not_descend_map[''].append('contrib')
+ public_api_visitor.do_not_descend_map['tf'].append('contrib')
  traverse.traverse(tf, public_api_visitor)
 
  proto_dict = visitor.GetProtos()

tensorflow/tools/common/public_api.py

@@ -36,12 +36,20 @@ def __init__(self, visitor):
  visitor: A visitor to call for the public API.
  """
  self._visitor = visitor
+ self._root_name = 'tf'
+
+ # Modules/classes we want to suppress entirely.
+ self._private_map = {
+ # Some implementations have this internal module that we shouldn't
+ # expose.
+ 'tf.flags': ['cpp_flags'],
+ }
 
  # Modules/classes we do not want to descend into if we hit them. Usually,
  # system modules exposed through platforms for compatibility reasons.
  # Each entry maps a module path to a name to ignore in traversal.
  self._do_not_descend_map = {
- '': [
+ 'tf': [
  'core',
  'examples',
  'flags', # Don't add flags
@@ -56,17 +64,25 @@ def __init__(self, visitor):
  'tensorboard',
  ],
 
- # Some implementations have this internal module that we shouldn't
- # expose.
- 'flags': ['cpp_flags'],
-
  ## Everything below here is legitimate.
  # It'll stay, but it's not officially part of the API.
- 'app': ['flags'],
+ 'tf.app': ['flags'],
  # Imported for compatibility between py2/3.
- 'test': ['mock'],
+ 'tf.test': ['mock'],
  }
 
+ @property
+ def private_map(self):
+ """A map from parents to symbols that should not be included at all.
+
+ This map can be edited, but it should not be edited once traversal has
+ begun.
+
+ Returns:
+ The map marking symbols to not include.
+ """
+ return self._private_map
+
  @property
  def do_not_descend_map(self):
  """A map from parents to symbols that should not be descended into.
@@ -79,11 +95,17 @@ def do_not_descend_map(self):
  """
  return self._do_not_descend_map
 
- def _isprivate(self, name):
+ def set_root_name(self, root_name):
+ """Override the default root name of 'tf'."""
+ self._root_name = root_name
+
+ def _is_private(self, path, name):
  """Return whether a name is private."""
  # TODO(wicke): Find out what names to exclude.
- return (name.startswith('_') and not re.match('__.*__$', name) or
- name in ['__base__', '__class__'])
+ return ((path in self._private_map and
+ name in self._private_map[path]) or
+ (name.startswith('_') and not re.match('__.*__$', name) or
+ name in ['__base__', '__class__']))
 
  def _do_not_descend(self, path, name):
  """Safely queries if a specific fully qualified name should be excluded."""
@@ -95,17 +117,21 @@ def __call__(self, path, parent, children):
 
  # Avoid long waits in cases of pretty unambiguous failure.
  if tf_inspect.ismodule(parent) and len(path.split('.')) > 10:
- raise RuntimeError('Modules nested too deep:\n%s\n\nThis is likely a '
- 'problem with an accidental public import.' % path)
+ raise RuntimeError('Modules nested too deep:\n%s.%s\n\nThis is likely a '
+ 'problem with an accidental public import.' %
+ (self._root_name, path))
+
+ # Includes self._root_name
+ full_path = '.'.join([self._root_name, path]) if path else self._root_name
 
  # Remove things that are not visible.
  for name, child in list(children):
- if self._isprivate(name):
+ if self._is_private(full_path, name):
  children.remove((name, child))
 
  self._visitor(path, parent, children)
 
  # Remove things that are visible, but which should not be descended into.
  for name, child in list(children):
- if self._do_not_descend(path, name):
+ if self._do_not_descend(full_path, name):
  children.remove((name, child))

tensorflow/tools/docs/doc_generator_visitor.py

@@ -30,7 +30,7 @@ def __init__(self, root_name=''):
  """Make a visitor.
 
  As this visitor is starting its traversal at a module or class, it will not
- be old the name of that object during traversal. `root_name` is the name it
+ be told the name of that object during traversal. `root_name` is the name it
  should use for that object, effectively prefixing all names with
  "root_name.".
 

tensorflow/tools/docs/generate_1_0.py

@@ -47,8 +47,8 @@
  doc_generator.set_py_modules([('tf', tf), ('tfdbg', tf_debug)])
 
  doc_generator.set_do_not_descend_map({
- '': ['cli', 'lib', 'wrappers'],
- 'contrib': [
+ 'tf': ['cli', 'lib', 'wrappers'],
+ 'tf.contrib': [
  'compiler',
  'factorization',
  'grid_rnn',
@@ -65,18 +65,18 @@
  'training',
  'tfprof',
  ],
- 'contrib.bayesflow': [
+ 'tf.contrib.bayesflow': [
  'entropy', 'monte_carlo', 'special_math',
  'stochastic_gradient_estimators', 'stochastic_graph',
  'stochastic_tensor', 'stochastic_variables', 'variational_inference'
  ],
- 'contrib.distributions': ['bijector'],
- 'contrib.ffmpeg': ['ffmpeg_ops'],
- 'contrib.graph_editor': [
+ 'tf.contrib.distributions': ['bijector'],
+ 'tf.contrib.ffmpeg': ['ffmpeg_ops'],
+ 'tf.contrib.graph_editor': [
  'edit', 'match', 'reroute', 'subgraph', 'transform', 'select', 'util'
  ],
- 'contrib.layers': ['feature_column', 'summaries'],
- 'contrib.learn': [
+ 'tf.contrib.layers': ['feature_column', 'summaries'],
+ 'tf.contrib.learn': [
  'datasets',
  'head',
  'graph_actions',
@@ -87,7 +87,7 @@
  'preprocessing',
  'utils',
  ],
- 'contrib.util': ['loader'],
+ 'tf.contrib.util': ['loader'],
  })
 
  sys.exit(doc_generator.build(flags))

tensorflow/tools/docs/generate_lib.py

@@ -180,11 +180,16 @@ def add_dict_to_dict(add_from, add_to):
 
 
 # Exclude some libaries in contrib from the documentation altogether.
+def _get_default_private_map():
+ return {}
+
+
+# Exclude members of some libaries.
 def _get_default_do_not_descend_map():
- # TODO(wicke): Shrink this list.
+ # TODO(wicke): Shrink this list once the modules get sealed.
  return {
- '': ['cli', 'lib', 'wrappers'],
- 'contrib': [
+ 'tf': ['cli', 'lib', 'wrappers'],
+ 'tf.contrib': [
  'compiler',
  'factorization',
  'grid_rnn',
@@ -200,17 +205,17 @@ def _get_default_do_not_descend_map():
  'testing',
  'tfprof',
  ],
- 'contrib.bayesflow': [
+ 'tf.contrib.bayesflow': [
  'special_math', 'stochastic_gradient_estimators',
  'stochastic_variables'
  ],
- 'contrib.ffmpeg': ['ffmpeg_ops'],
- 'contrib.graph_editor': [
+ 'tf.contrib.ffmpeg': ['ffmpeg_ops'],
+ 'tf.contrib.graph_editor': [
  'edit', 'match', 'reroute', 'subgraph', 'transform', 'select', 'util'
  ],
- 'contrib.keras': ['api', 'python'],
- 'contrib.layers': ['feature_column', 'summaries'],
- 'contrib.learn': [
+ 'tf.contrib.keras': ['api', 'python'],
+ 'tf.contrib.layers': ['feature_column', 'summaries'],
+ 'tf.contrib.learn': [
  'datasets',
  'head',
  'graph_actions',
@@ -221,22 +226,25 @@ def _get_default_do_not_descend_map():
  'preprocessing',
  'utils',
  ],
- 'contrib.util': ['loader'],
+ 'tf.contrib.util': ['loader'],
  }
 
 
-def extract(py_modules, do_not_descend_map):
+def extract(py_modules, private_map, do_not_descend_map):
  """Extract docs from tf namespace and write them to disk."""
  # Traverse the first module.
  visitor = doc_generator_visitor.DocGeneratorVisitor(py_modules[0][0])
  api_visitor = public_api.PublicAPIVisitor(visitor)
+ api_visitor.set_root_name(py_modules[0][0])
+ add_dict_to_dict(private_map, api_visitor.private_map)
  add_dict_to_dict(do_not_descend_map, api_visitor.do_not_descend_map)
 
  traverse.traverse(py_modules[0][1], api_visitor)
 
  # Traverse all py_modules after the first:
  for module_name, module in py_modules[1:]:
  visitor.set_root_name(module_name)
+ api_visitor.set_root_name(module_name)
  traverse.traverse(module, api_visitor)
 
  return visitor
@@ -409,6 +417,7 @@ class DocGenerator(object):
  def __init__(self):
  self.argument_parser = argparse.ArgumentParser()
  self._py_modules = None
+ self._private_map = _get_default_private_map()
  self._do_not_descend_map = _get_default_do_not_descend_map()
  self.yaml_toc = True
 
@@ -439,9 +448,15 @@ def parse_known_args(self):
  flags, _ = self.argument_parser.parse_known_args()
  return flags
 
+ def add_to_private_map(self, d):
+ add_dict_to_dict(d, self._private_map)
+
  def add_to_do_not_descend_map(self, d):
  add_dict_to_dict(d, self._do_not_descend_map)
 
+ def set_private_map(self, d):
+ self._private_map = d
+
  def set_do_not_descend_map(self, d):
  self._do_not_descend_map = d
 
@@ -471,7 +486,8 @@ def make_parser_config(self, visitor, reference_resolver, guide_index,
  base_dir=base_dir)
 
  def run_extraction(self):
- return extract(self._py_modules, self._do_not_descend_map)
+ return extract(
+ self._py_modules, self._private_map, self._do_not_descend_map)
 
  def build(self, flags):
  """Actually build the docs."""

tensorflow/tools/docs/generate_lib_test.py

@@ -58,8 +58,9 @@ def test_extraction(self):
  py_modules = [('tf', tf), ('tfdbg', tf_debug)]
 
  try:
- generate_lib.extract(
- py_modules, generate_lib._get_default_do_not_descend_map())
+ generate_lib.extract(py_modules,
+ generate_lib._get_default_private_map(),
+ generate_lib._get_default_do_not_descend_map())
  except RuntimeError:
  print('*****************************************************************')
  print('If this test fails, you have most likely introduced an unsealed')