On demand indexing

apps/els_core/src/els_config.erl

@@ -267,8 +267,8 @@ consult_config([Path | Paths], ReportMissingConfig) ->
  [Config] -> {Path, Config}
  catch
  Class:Error ->
- ?LOG_WARNING( "Could not read config file: path=~p class=~p error=~p"
-  , [Path, Class, Error]),
+ ?LOG_DEBUG( "Could not read config file: path=~p class=~p error=~p"
+ , [Path, Class, Error]),
  consult_config(Paths, ReportMissingConfig)
  end.
 

apps/els_core/src/els_utils.erl

@@ -154,9 +154,9 @@ lookup_document(Uri) ->
  case els_dt_document:lookup(Uri) of
  {ok, [Document]} ->
  {ok, Document};
- Error ->
- ?LOG_INFO("Document lookup failed [error=~p] [uri=~p]", [Error, Uri]),
- {error, Error}
+ {ok, []} ->
+ ?LOG_INFO("Document lookup failed [uri=~p]", [Uri]),
+ {error, document_lookup_failed}
  end
  end.
 

apps/els_lsp/priv/code_navigation/src/purge_references.erl

@@ -0,0 +1,5 @@
+-module(purge_references).
+
+-spec foo(any()) -> ok.
+foo(_X) -> ok.
+bar() -> foo(42).

apps/els_lsp/src/els_dt_document.erl

@@ -53,7 +53,7 @@
  , kind :: kind() | '_'
  , text :: binary() | '_'
  , md5 :: binary() | '_'
- , pois :: [poi()] | '_'
+ , pois :: [poi()] | '_' | ondemand
  }).
 -type els_dt_document() :: #els_dt_document{}.
 
@@ -62,7 +62,7 @@
  , kind := kind()
  , text := binary()
  , md5 => binary()
- , pois => [poi()]
+ , pois => [poi()] | ondemand
  }.
 -export_type([ id/0
  , item/0
@@ -145,18 +145,21 @@ new(Uri, Text) ->
 
 -spec new(uri(), binary(), atom(), kind()) -> item().
 new(Uri, Text, Id, Kind) ->
- {ok, POIs} = els_parser:parse(Text),
- MD5 = erlang:md5(Text),
+ MD5 = erlang:md5(Text),
  #{ uri => Uri
  , id => Id
  , kind => Kind
  , text => Text
  , md5 => MD5
- , pois => POIs
+ , pois => ondemand
  }.
 
 %% @doc Returns the list of POIs for the current document
 -spec pois(item()) -> [poi()].
+pois(#{ text := Text, pois := ondemand } = Document) ->
+ {ok, POIs} = els_parser:parse(Text),
+ ok = els_dt_document:insert(Document#{pois => POIs}),
+ POIs;
 pois(#{ pois := POIs }) ->
  POIs.
 

apps/els_lsp/src/els_dt_references.erl

@@ -18,16 +18,23 @@
 %%==============================================================================
 
 -export([ delete_by_uri/1
- , find_all/0
  , find_by/1
  , find_by_id/2
  , insert/2
  ]).
 
+%%==============================================================================
+%% Test API
+%%==============================================================================
+
+-export([ find_candidate_uris/1
+ ]).
+
 %%==============================================================================
 %% Includes
 %%==============================================================================
 -include("els_lsp.hrl").
+-include_lib("kernel/include/logger.hrl").
 
 %%==============================================================================
 %% Item Definition
@@ -45,6 +52,14 @@
  }.
 -export_type([ item/0 ]).
 
+-type poi_category() :: function
+ | type
+ | macro
+ | record
+ | include
+ | include_lib
+ | behaviour.
+
 %%==============================================================================
 %% Callbacks for the els_db_table Behaviour
 %%==============================================================================
@@ -82,12 +97,6 @@ insert(Kind, Map) when is_map(Map) ->
  Record = from_item(Kind, Map),
  els_db:write(name(), Record).
 
-%% @doc Find all
--spec find_all() -> {ok, [item()]} | {error, any()}.
-find_all() ->
- Pattern = #els_dt_references{_ = '_'},
- find_by(Pattern).
-
 %% @doc Find by id
 -spec find_by_id(poi_kind(), any()) -> {ok, [item()]} | {error, any()}.
 find_by_id(Kind, Id) ->
@@ -97,10 +106,58 @@ find_by_id(Kind, Id) ->
 
 -spec find_by(tuple()) -> {ok, [item()]}.
 find_by(Pattern) ->
+ #els_dt_references{id = Id} = Pattern,
+ Uris = find_candidate_uris(Id),
+ [begin
+ {ok, Document} = els_utils:lookup_document(Uri),
+ POIs = els_dt_document:pois(Document, [ application
+ , behaviour
+ , implicit_fun
+ , include
+ , include_lib
+ , type_application
+ , import_entry
+ ]),
+ %% TODO: Only re-register references if necessary
+ ok = els_dt_references:delete_by_uri(Uri),
+ [register_reference(Uri, POI) || POI <- POIs]
+ end || Uri <- Uris],
  {ok, Items} = els_db:match(name(), Pattern),
  {ok, [to_item(Item) || Item <- Items]}.
 
--spec kind_to_category(poi_kind()) -> function | type | macro | record.
+%% TODO: What about references from header files?
+-spec greppable_string({poi_category(), any()}) ->
+ {ok, string()} | {error, {any(), not_supported}}.
+greppable_string({function, {_M, F, _A}}) ->
+ io_lib:format("~p", [F]);
+greppable_string({type, {_M, F, _A}}) ->
+ io_lib:format("~p", [F]);
+greppable_string({macro, {Name, _Arity}}) ->
+ io_lib:format("~p", [Name]);
+greppable_string({macro, Name}) ->
+ io_lib:format("~p", [Name]);
+greppable_string({include, String}) ->
+ io_lib:format("~s", [String]);
+%% TODO: This could be tricky
+greppable_string({include_lib, String}) ->
+ io_lib:format("~s", [String]);
+greppable_string({behaviour, Name}) ->
+ io_lib:format("~p", [Name]).
+
+-spec find_candidate_uris(any()) -> [uri()].
+find_candidate_uris(Id) ->
+ IdString = greppable_string(Id),
+ Paths = els_config:get(apps_paths) ++ els_config:get(deps_paths),
+ PathsString = string:join(Paths, " "),
+ Cmd = "grep -l -r \"" ++ IdString ++ "\" " ++ PathsString,
+ ?LOG_DEBUG("Command: ~p", [Cmd]),
+ Result = string:trim(os:cmd(Cmd), trailing),
+ ?LOG_DEBUG("Result: ~p", [Result]),
+ Candidates = string:split(Result, "\n", all),
+ [els_uri:uri(els_utils:to_binary(Candidate)) || Candidate <- Candidates,
+ Candidate =/= []].
+
+-spec kind_to_category(poi_kind()) -> poi_category().
 kind_to_category(Kind) when Kind =:= application;
  Kind =:= export_entry;
  Kind =:= function;
@@ -124,3 +181,23 @@ kind_to_category(Kind) when Kind =:= include_lib ->
  include_lib;
 kind_to_category(Kind) when Kind =:= behaviour ->
  behaviour.
+
+-spec register_reference(uri(), poi()) -> ok.
+register_reference(Uri, #{id := {F, A}} = POI) ->
+ M = els_uri:module(Uri),
+ register_reference(Uri, POI#{id => {M, F, A}});
+register_reference(Uri, #{kind := Kind, id := Id, range := Range})
+ when %% Include
+ Kind =:= include;
+ Kind =:= include_lib;
+ %% Function
+ Kind =:= application;
+ Kind =:= implicit_fun;
+ Kind =:= import_entry;
+ %% Type
+ Kind =:= type_application;
+ %% Behaviour
+ Kind =:= behaviour ->
+ insert( Kind
+ , #{id => Id, uri => Uri, range => Range}
+ ).

apps/els_lsp/src/els_dt_signatures.erl

@@ -72,9 +72,24 @@ insert(Map) when is_map(Map) ->
  els_db:write(name(), Record).
 
 -spec lookup(mfa()) -> {ok, [item()]}.
-lookup(MFA) ->
- {ok, Items} = els_db:lookup(name(), MFA),
- {ok, [to_item(Item) || Item <- Items]}.
+lookup({M, _F, _A} = MFA) ->
+ %% TODO: Only do it when necessary
+ case els_utils:find_module(M) of
+ {ok, Uri} ->
+ {ok, #{text := Text} = Document} = els_utils:lookup_document(Uri),
+ Specs = els_dt_document:pois(Document, [spec]),
+ [ begin
+ #{from := From, to := To} = Range,
+ Spec = els_text:range(Text, From, To),
+ els_dt_signatures:insert(#{ mfa => {M, F, A} , spec => Spec})
+ end
+ || #{id := {F, A}, range := Range} <- Specs
+ ],
+ {ok, Items} = els_db:lookup(name(), MFA),
+ {ok, [to_item(Item) || Item <- Items]};
+ {error, _} ->
+ {ok, []}
+ end.
 
 -spec delete_by_module(atom()) -> ok.
 delete_by_module(Module) ->

apps/els_lsp/src/els_indexing.erl

@@ -80,57 +80,14 @@ index(Uri, Text, Mode) ->
  ok;
  {ok, LookupResult} ->
  Document = els_dt_document:new(Uri, Text),
+ ok = els_dt_document:insert(Document),
  do_index(Document, Mode, LookupResult =/= [])
  end.
 
 -spec do_index(els_dt_document:item(), mode(), boolean()) -> ok.
-do_index(#{uri := Uri, id := Id, kind := Kind} = Document, Mode, Reset) ->
- case Mode of
- 'deep' ->
- ok = els_dt_document:insert(Document);
- 'shallow' ->
- %% Don't store detailed POIs when "shallow" indexing.
- %% They will be reloaded and inserted when needed
- %% by calling els_utils:lookup_document/1
- ok
- end,
- %% Mapping from document id to uri
+do_index(#{uri := Uri, id := Id, kind := Kind}, _Mode, _Reset) ->
  ModuleItem = els_dt_document_index:new(Id, Uri, Kind),
- ok = els_dt_document_index:insert(ModuleItem),
- index_signatures(Document),
- index_references(Document, Mode, Reset).
-
--spec index_signatures(els_dt_document:item()) -> ok.
-index_signatures(#{id := Id, text := Text} = Document) ->
- Specs = els_dt_document:pois(Document, [spec]),
- [ begin
- #{from := From, to := To} = Range,
- Spec = els_text:range(Text, From, To),
- els_dt_signatures:insert(#{ mfa => {Id, F, A} , spec => Spec})
- end
- || #{id := {F, A}, range := Range} <- Specs
- ],
- ok.
-
--spec index_references(els_dt_document:item(), mode(), boolean()) -> ok.
-index_references(#{uri := Uri} = Document, 'deep', true) ->
- %% Optimization to only do (non-optimized) match_delete when necessary
- ok = els_dt_references:delete_by_uri(Uri),
- index_references(Document, 'deep', false);
-index_references(#{uri := Uri} = Document, 'deep', false) ->
- %% References
- POIs = els_dt_document:pois(Document, [ application
- , behaviour
- , implicit_fun
- , include
- , include_lib
- , type_application
- , import_entry
- ]),
- [register_reference(Uri, POI) || POI <- POIs],
- ok;
-index_references(_Document, 'shallow', _) ->
- ok.
+ ok = els_dt_document_index:insert(ModuleItem).
 
 -spec maybe_start() -> true | false.
 maybe_start() ->
@@ -194,26 +151,6 @@ try_index_file(FullName, Mode, SkipGeneratedFiles, GeneratedFilesTag) ->
  {error, {Type, Reason}}
  end.
 
--spec register_reference(uri(), poi()) -> ok.
-register_reference(Uri, #{id := {F, A}} = POI) ->
- M = els_uri:module(Uri),
- register_reference(Uri, POI#{id => {M, F, A}});
-register_reference(Uri, #{kind := Kind, id := Id, range := Range})
- when %% Include
- Kind =:= include;
- Kind =:= include_lib;
- %% Function
- Kind =:= application;
- Kind =:= implicit_fun;
- Kind =:= import_entry;
- %% Type
- Kind =:= type_application;
- %% Behaviour
- Kind =:= behaviour ->
- els_dt_references:insert( Kind
- , #{id => Id, uri => Uri, range => Range}
- ).
-
 -spec index_dir(string(), mode()) ->
  {non_neg_integer(), non_neg_integer(), non_neg_integer()}.
 index_dir(Dir, Mode) ->

apps/els_lsp/test/els_call_hierarchy_SUITE.erl

@@ -91,7 +91,36 @@ incoming_calls(Config) ->
  , uri => UriA},
  ?assertEqual([Item], PrepareResult),
  #{result := Result} = els_client:callhierarchy_incomingcalls(Item),
- Calls = [#{ from =>
+ Calls = [ #{ from =>
+ #{ data =>
+ els_utils:base64_encode_term(
+ #{ poi =>
+ #{ data =>
+ #{ args => [{1, "Arg1"}]
+ , wrapping_range =>
+ #{ from => {7, 1}
+ , to => {14, 0}}}
+ , id => {function_a, 1}
+ , kind => function
+ , range => #{from => {7, 1}, to => {7, 11}}}})
+ , detail => <<"call_hierarchy_b [L11]">>
+ , kind => 12
+ , name => <<"function_a/1">>
+ , range =>
+ #{ 'end' => #{character => 29, line => 10}
+ , start => #{character => 2, line => 10}
+ }
+ , selectionRange =>
+ #{ 'end' => #{character => 29, line => 10}
+ , start => #{character => 2, line => 10}
+ }
+ , uri => UriB}
+ , fromRanges =>
+ [#{ 'end' => #{character => 29, line => 10}
+ , start => #{character => 2, line => 10}
+ }]
+ }
+ , #{ from =>
  #{ data =>
  els_utils:base64_encode_term(
  #{ poi =>
@@ -134,35 +163,6 @@ incoming_calls(Config) ->
  [#{ 'end' => #{character => 12, line => 15}
  , start => #{character => 2, line => 15}
  }]}
- , #{ from =>
- #{ data =>
- els_utils:base64_encode_term(
- #{ poi =>
- #{ data =>
- #{ args => [{1, "Arg1"}]
- , wrapping_range =>
- #{ from => {7, 1}
- , to => {14, 0}}}
- , id => {function_a, 1}
- , kind => function
- , range => #{from => {7, 1}, to => {7, 11}}}})
- , detail => <<"call_hierarchy_b [L11]">>
- , kind => 12
- , name => <<"function_a/1">>
- , range =>
- #{ 'end' => #{character => 29, line => 10}
- , start => #{character => 2, line => 10}
- }
- , selectionRange =>
- #{ 'end' => #{character => 29, line => 10}
- , start => #{character => 2, line => 10}
- }
- , uri => UriB}
- , fromRanges =>
- [#{ 'end' => #{character => 29, line => 10}
- , start => #{character => 2, line => 10}
- }]
- }
  ],
  ?assertEqual(Calls, Result).