issue viewvc#216: More tweaks to binary file handling

* lib/config.py,
* conf/viewvc.conf.dist
  (hide_binary_content): Remove in favor of ...
  (allow_mojibake): ...this new option.

* lib/common.py
  (TemplateData.__str__): New.

* lib/viewvc.py
  Handle binary files gracefully now, with the is_binary template
  variable.  Check allow_mojibake option now instead of
  hide_binary_content.

* CHANGES,
* docs/upgrading-howto.html
* docs/template-authoring-guide.html
  Document all this binary handling stuff.

CHANGES

@@ -5,6 +5,8 @@ Version 1.3.0 (released ??-???-????)
  - minimum Pygments version now 1.1
  * make-database now requires --force to destroy existing DB (#212)
  * removed deprecated checkout_magic option/behavior (#215)
+ * new 'allow_mojibake' option (#216)
+ * require explicit match for web-friendly images in 'binary_mime_types' (#216)
 
 Version 1.2.1 (released 26-Mar-2020)
 

conf/viewvc.conf.dist

@@ -402,22 +402,24 @@
 ##
 #allowed_views = annotate, diff, markup, roots
 
-## hide_binary_content: Hide the contents of binary files:
-## 1 Hide binary file contents
-## 0 Show binary file contents (as mojibake).
-##
-## ViewVC attempts to transcode file content to Unicode for display,
-## and does so using all the hints at its disposal for the file's
-## source encoding (see the "detect_encoding", "default_encoding", and
-## "svn_ignore_mimetype" options). But sometimes this process fails,
-## typically because the file's contents aren't actually intended to
-## be human-readable (a so-called "binary file").
-##
-## If you wish to force ViewVC to consider certain file MIME content
-## types as binary regardless of the actual content, see the
-## "binary_mime_types" option.
-##
-#hide_binary_content = 1
+## allow_mojibake: Force the (lossy) display of ostensibly
+## human-readable file content when Unicode conversion fails.
+##
+## ViewVC attempts to transcode the content of versioned files to
+## Unicode for display, and does so using all the hints at its
+## disposal for the file's source encoding (see the "detect_encoding",
+## "default_encoding", and "svn_ignore_mimetype" options). But
+## sometimes this process fails, either because the file's contents
+## aren't actually intended to be human-readable (a so-called "binary
+## file") or because the source encoding guesswork just gets it wrong.
+## Enable this feature to force ViewVC to perform a lossy conversion
+## to Unicode and display those results to the user.
+##
+## Note that this feature applies only to files that ViewVC hasn't
+## otherwise already determined as "binary" (per the
+## 'hide_binary_content', for example).
+##
+#allow_mojibake = 0
 
 ## binary_mime_types: Comma-delimited list of MIME content types (with
 ## support for fnmatch-style glob characters) which are always

docs/template-authoring-guide.html

@@ -488,6 +488,12 @@ <h3 id="variables-file">File Contents View (file.ezt)</h3>
  <td>Date (in UTC if not otherwise configured) of the revision currently
  being viewed.</td>
 </tr>
+<tr class="varlevel1">
+ <td class="varname">is_binary</td>
+ <td>Boolean</td>
+ <td>Indicates whether the file's contents are not deemed to be
+ "human-readable".</td>
+</tr>
 <tr class="varlevel1">
  <td class="varname">image_src_href</td>
  <td>String</td>

docs/upgrading-howto.html

@@ -93,6 +93,41 @@ <h2 id="sec-from-1-2">Upgrading From ViewVC 1.2</h2>
 
 <p>This section discusses how to upgrade ViewVC 1.2.x to ViewVC 1.3.x.</p>
 
+<!-- ------------------------------------------------------------------------ -->
+<div class="h3">
+<h3>Binary File Handling</h3>
+
+<p>The <var>options/binary_mime_types</var> option has changed
+ slightly in that glob patterns (such as "image/*") no longer apply
+ to the three "web-friendly" image formats. Administrators who
+ wish to forbid viewing of these images types ("image/gif",
+ "image/jpeg", and "image/png") should add them explicitly to this
+ option's value (in addition, if you wish, to a blanket "image/*"
+ that covers all other image types). Of course, if your
+ installation does not allow the use of the "co" (checkout or
+ download) view, embedded web-friendly images will not be displayed
+ even if not explicitly forbidden by this option.</p>
+
+<p>Additionally, what ViewVC <em>does</em> with binary files has
+ changed. Rather than raising a page-level exception on a request
+ to view a binary file's content or diff, ViewVC will avoid showing
+ the deemed-to-be-unreadable file content but still preserve its own
+ page layout and navigation.</p>
+
+<p>Finally, recognizing the challenges of handling even human-readable
+ file content (which as of this release implies an ability to
+ perfectly represent that content as Unicode), we've added a new
+ <var>options/allow_mojibake</var> option that administrators can
+ enable in order to force ViewVC to fall back to a lossy Unicode
+ transformation and present those results to the user. This is
+ rougly analogous to the old <var>hide_binary_garbage</var>
+ template-defined flag that was present in some previous ViewVC
+ template sets. Note that this option does not override other
+ determinations of "binariness" (such as the application
+ of <var>options/binary_mime_types</var>.</p>
+
+</div>
+
 <!-- ------------------------------------------------------------------------ -->
 <div class="h3">
 <h3>Configuration Options</h3>
@@ -103,7 +138,7 @@ <h3>Configuration Options</h3>
 <p>The following configuration options were removed:</p>
 
 <ul>
- <li>options/checkout_magic</li>
+ <li><var>options/checkout_magic</var></li>
 </ul>
 
 </div>

lib/common.py

@@ -48,6 +48,9 @@ def __setitem__(self, key, item):
  def __delitem__(self, key):
  return self._items.__delitem__(key)
 
+ def __str__(self):
+ return str(self._items)
+
  def keys(self):
  return self._items.keys()
 

lib/config.py

@@ -407,7 +407,7 @@ def set_defaults(self):
  self.options.mangle_email_addresses = 0
  self.options.custom_log_formatting = []
  self.options.default_file_view = "log"
- self.options.hide_binary_content = 1
+ self.options.allow_mojibake = 0
  self.options.binary_mime_types = []
  self.options.dir_ignored_files = []
  self.options.http_expiration_time = 600

lib/viewvc.py

@@ -1980,13 +1980,18 @@ def markup_or_annotate(request, is_annotate):
 
  # Is this display blocked by 'binary_mime_types' configuration?
  if is_binary_file_mime_type(mime_type, cfg):
- raise debug.ViewVCException('Display of binary file content disabled '
- 'by configuration', '403 Forbidden')
+ fp, revision = request.repos.openfile(path, rev, {})
+ fp.close()
+ if check_freshness(request, None, revision, weak=1):
+ return
+ is_binary = True
+ if is_annotate:
+ annotation = 'binary'
 
  # If this is viewable image that we're allowed to show embedded, we
  # need only resolve its revision and generate an image src=
  # attribute URL for it.
- if is_viewable_image(mime_type) and 'co' in cfg.options.allowed_views:
+ elif is_viewable_image(mime_type) and 'co' in cfg.options.allowed_views:
  fp, revision = request.repos.openfile(path, rev, {})
  fp.close()
  if check_freshness(request, None, revision, weak=1):
@@ -2074,7 +2079,7 @@ def markup_or_annotate(request, is_annotate):
  try:
  line = line.decode(encoding)
  except UnicodeDecodeError:
- if cfg.options.hide_binary_content:
+ if not cfg.options.allow_mojibake:
  raise
  line = line.decode(encoding, 'surrogateescape')
  file_lines[i] = line