Use Ejson Body Instead of Compressed Body for External size

src/couch/src/couch_db.erl

@@ -1111,7 +1111,11 @@ prepare_doc_summaries(Db, BucketList) ->
  nil
  end,
  SummaryChunk = couch_db_updater:make_doc_summary(Db, {Body, DiskAtts}),
- Doc#doc{body = {summary, SummaryChunk, SizeInfo, AttsFd}}
+ Meta = Doc#doc.meta,
+ Doc#doc{
+ body = {summary, SummaryChunk, SizeInfo, AttsFd},
+ meta = [{ejson_size, ?term_size(Body)} | Meta]
+ }
  end,
  Bucket) || Bucket <- BucketList].
 

src/couch/src/couch_db_updater.erl

@@ -677,6 +677,7 @@ flush_trees(#db{fd = Fd} = Db,
  case Value of
  #doc{deleted = IsDeleted, body = {summary, _, _, _} = DocSummary} ->
  {summary, Summary, AttSizeInfo, AttsFd} = DocSummary,
+ ExternalSize = get_meta_body_size(Value#doc.meta, Summary),
  % this node value is actually an unwritten document summary,
  % write to disk.
  % make sure the Fd in the written bins is the same Fd we are
@@ -695,7 +696,6 @@ flush_trees(#db{fd = Fd} = Db,
  " changed. Possibly retrying.", []),
  throw(retry)
  end,
- ExternalSize = ?term_size(Summary),
  {ok, NewSummaryPointer, SummarySize} =
  couch_file:append_raw_chunk(Fd, Summary),
  Leaf = #leaf{
@@ -1086,8 +1086,16 @@ copy_docs(Db, #db{fd = DestFd} = NewDb, MixedInfos, Retry) ->
  {NewRevTree, FinalAcc} = couch_key_tree:mapfold(fun
  (_Rev, #leaf{ptr=Sp}=Leaf, leaf, SizesAcc) ->
  {Body, AttInfos} = copy_doc_attachments(Db, Sp, DestFd),
+ % In the future, we should figure out how to do this for
+ % upgrade purposes.
+ EJsonBody = case is_binary(Body) of
+ true ->
+ couch_compress:decompress(Body);
+ false ->
+ Body
+ end,
  SummaryChunk = make_doc_summary(NewDb, {Body, AttInfos}),
- ExternalSize = ?term_size(SummaryChunk),
+ ExternalSize = ?term_size(EJsonBody),
  {ok, Pos, SummarySize} = couch_file:append_raw_chunk(
  DestFd, SummaryChunk),
  AttSizes = [{element(3,A), element(4,A)} || A <- AttInfos],
@@ -1467,6 +1475,16 @@ make_doc_summary(#db{compression = Comp}, {Body0, Atts0}) ->
  SummaryBin = ?term_to_bin({Body, Atts}),
  couch_file:assemble_file_chunk(SummaryBin, couch_crypto:hash(md5, SummaryBin)).
 
+
+get_meta_body_size(Meta, Summary) ->
+ case lists:keyfind(ejson_size, 1, Meta) of
+ {ejson_size, ExternalSize} ->
+ ExternalSize;
+ false ->
+ ?term_size(couch_compress:decompress(Summary))
+ end.
+
+
 default_security_object(<<"shards/", _/binary>>) ->
  case config:get("couchdb", "default_security", "everyone") of
  "admin_only" ->

src/couch/test/couchdb_file_compression_tests.erl

@@ -111,16 +111,19 @@ should_compare_compression_methods(DbName) ->
 
 compare_compression_methods(DbName) ->
  config:set("couchdb", "file_compression", "none", false),
+ ExternalSizePreCompact = db_external_size(DbName),
  compact_db(DbName),
  compact_view(DbName),
  DbSizeNone = db_disk_size(DbName),
  ViewSizeNone = view_disk_size(DbName),
+ ExternalSizeNone = db_external_size(DbName),
 
  config:set("couchdb", "file_compression", "snappy", false),
  compact_db(DbName),
  compact_view(DbName),
  DbSizeSnappy = db_disk_size(DbName),
  ViewSizeSnappy = view_disk_size(DbName),
+ ExternalSizeSnappy = db_external_size(DbName),
 
  ?assert(DbSizeNone > DbSizeSnappy),
  ?assert(ViewSizeNone > ViewSizeSnappy),
@@ -139,9 +142,13 @@ compare_compression_methods(DbName) ->
  compact_view(DbName),
  DbSizeDeflate9 = db_disk_size(DbName),
  ViewSizeDeflate9 = view_disk_size(DbName),
+ ExternalSizeDeflate9 = db_external_size(DbName),
 
  ?assert(DbSizeDeflate1 > DbSizeDeflate9),
- ?assert(ViewSizeDeflate1 > ViewSizeDeflate9).
+ ?assert(ViewSizeDeflate1 > ViewSizeDeflate9),
+ ?assert(ExternalSizePreCompact =:= ExternalSizeNone),
+ ?assert(ExternalSizeNone =:= ExternalSizeSnappy),
+ ?assert(ExternalSizeNone =:= ExternalSizeDeflate9).
 
 
 populate_db(_Db, NumDocs) when NumDocs =< 0 ->
@@ -186,6 +193,12 @@ db_disk_size(DbName) ->
  ok = couch_db:close(Db),
  active_size(Info).
 
+db_external_size(DbName) ->
+ {ok, Db} = couch_db:open_int(DbName, []),
+ {ok, Info} = couch_db:get_db_info(Db),
+ ok = couch_db:close(Db),
+ external_size(Info).
+
 view_disk_size(DbName) ->
  {ok, Db} = couch_db:open_int(DbName, []),
  {ok, DDoc} = couch_db:open_doc(Db, ?DDOC_ID, [ejson_body]),
@@ -196,6 +209,9 @@ view_disk_size(DbName) ->
 active_size(Info) ->
  couch_util:get_nested_json_value({Info}, [sizes, active]).
 
+external_size(Info) ->
+ couch_util:get_nested_json_value({Info}, [sizes, external]).
+
 wait_compaction(DbName, Kind, Line) ->
  WaitFun = fun() ->
  case is_compaction_running(DbName) of