On demand indexing (erlang-ls#1260)

apps/els_core/src/els_config.erl

@@ -273,8 +273,8 @@ consult_config([Path | Paths], ReportMissingConfig) ->
  [Config] -> {Path, Config}
  catch
  Class:Error ->
- ?LOG_WARNING( "Could not read config file: path=~p class=~p error=~p"
-  , [Path, Class, Error]),
+ ?LOG_DEBUG( "Could not read config file: path=~p class=~p error=~p"
+ , [Path, Class, Error]),
  consult_config(Paths, ReportMissingConfig)
  end.
 

apps/els_core/src/els_utils.erl

@@ -110,7 +110,7 @@ find_header(Id) ->
  {ok, Uri};
  [] ->
  FileName = atom_to_list(Id) ++ ".hrl",
- els_indexing:find_and_index_file(FileName)
+ els_indexing:find_and_deeply_index_file(FileName)
  end.
 
 %% @doc Look for a module in the DB
@@ -128,14 +128,14 @@ find_module(Id) ->
 find_modules(Id) ->
  {ok, Candidates} = els_dt_document_index:lookup(Id),
  case [Uri || #{kind := module, uri := Uri} <- Candidates] of
-  [] ->
-  FileName = atom_to_list(Id) ++ ".erl",
-  case els_indexing:find_and_index_file(FileName) of
-  {ok, Uri} -> {ok, [Uri]};
-  Error -> Error
-  end;
-  Uris ->
-  {ok, prioritize_uris(Uris)}
+ [] ->
+ FileName = atom_to_list(Id) ++ ".erl",
+ case els_indexing:find_and_deeply_index_file(FileName) of
+ {ok, Uri} -> {ok, [Uri]};
+ Error -> Error
+ end;
+ Uris ->
+ {ok, prioritize_uris(Uris)}
  end.
 
 %% @doc Look for a document in the DB.
@@ -150,13 +150,13 @@ lookup_document(Uri) ->
  {ok, Document};
  {ok, []} ->
  Path = els_uri:path(Uri),
- {ok, Uri} = els_indexing:index_file(Path),
+ {ok, Uri} = els_indexing:shallow_index(Path, app),
  case els_dt_document:lookup(Uri) of
  {ok, [Document]} ->
  {ok, Document};
- Error ->
- ?LOG_INFO("Document lookup failed [error=~p] [uri=~p]", [Error, Uri]),
- {error, Error}
+ {ok, []} ->
+ ?LOG_INFO("Document lookup failed [uri=~p]", [Uri]),
+ {error, document_lookup_failed}
  end
  end.
 

apps/els_lsp/src/els_background_job.erl

@@ -187,13 +187,21 @@ terminate(normal, #{ config := #{on_complete := OnComplete}
  ?LOG_DEBUG("Background job completed.", []),
  OnComplete(InternalState),
  ok;
-terminate(Reason, #{ config := #{on_error := OnError}
+terminate(Reason, #{ config := #{ on_error := OnError
+ , title := Title
+ }
  , internal_state := InternalState
  , token := Token
  , total := Total
  , progress_enabled := ProgressEnabled
  }) ->
- ?LOG_WARNING( "Background job aborted. [reason=~p]", [Reason]),
+ case Reason of
+ shutdown ->
+ ?LOG_DEBUG("Background job terminated.", []);
+ _ ->
+ ?LOG_ERROR( "Background job aborted. [reason=~p] [title=~p",
+ [Reason, Title])
+ end,
  notify_end(Token, Total, ProgressEnabled),
  OnError(InternalState),
  ok.

apps/els_lsp/src/els_buffer_server.erl

@@ -0,0 +1,126 @@
+%%%=============================================================================
+%%% @doc Buffer edits to an open buffer to avoid re-indexing too often.
+%%% @end
+%%%=============================================================================
+-module(els_buffer_server).
+
+%%==============================================================================
+%% API
+%%==============================================================================
+-export([ new/2
+ , stop/1
+ , apply_edits/2
+ , flush/1
+ ]).
+
+-export([ start_link/2 ]).
+
+%%==============================================================================
+%% Callbacks for the gen_server behaviour
+%%==============================================================================
+-behaviour(gen_server).
+-export([ init/1
+ , handle_call/3
+ , handle_cast/2
+ , handle_info/2
+ ]).
+
+%%==============================================================================
+%% Macro Definitions
+%%==============================================================================
+-define(FLUSH_DELAY, 200). %% ms
+
+%%==============================================================================
+%% Type Definitions
+%%==============================================================================
+-type text() :: binary().
+-type state() :: #{ uri := uri()
+ , text := text()
+ , ref := undefined | reference()
+ , pending := [{pid(), any()}]
+ }.
+-type buffer() :: pid().
+
+%%==============================================================================
+%% Includes
+%%==============================================================================
+-include_lib("kernel/include/logger.hrl").
+-include("els_lsp.hrl").
+
+%%==============================================================================
+%% API
+%%==============================================================================
+-spec new(uri(), text()) -> {ok, pid()}.
+new(Uri, Text) ->
+ supervisor:start_child(els_buffer_sup, [Uri, Text]).
+
+-spec stop(buffer()) -> ok.
+stop(Buffer) ->
+ supervisor:terminate_child(els_buffer_sup, Buffer).
+
+-spec apply_edits(buffer(), [els_text:edit()]) -> ok.
+apply_edits(Buffer, Edits) ->
+ gen_server:cast(Buffer, {apply_edits, Edits}).
+
+-spec flush(buffer()) -> text().
+flush(Buffer) ->
+ gen_server:call(Buffer, {flush}).
+
+-spec start_link(uri(), text()) -> {ok, buffer()}.
+start_link(Uri, Text) ->
+ gen_server:start_link(?MODULE, {Uri, Text}, []).
+
+%%==============================================================================
+%% Callbacks for the gen_server behaviour
+%%==============================================================================
+-spec init({uri(), text()}) -> {ok, state()}.
+init({Uri, Text}) ->
+ {ok, #{ uri => Uri, text => Text, ref => undefined, pending => [] }}.
+
+-spec handle_call(any(), {pid(), any()}, state()) -> {reply, any(), state()}.
+handle_call({flush}, From, State) ->
+ #{uri := Uri, ref := Ref0, pending := Pending0} = State,
+ ?LOG_DEBUG("[~p] Flushing request [uri=~p]", [?MODULE, Uri]),
+ cancel_flush(Ref0),
+ Ref = schedule_flush(),
+ {noreply, State#{ref => Ref, pending => [From|Pending0]}};
+handle_call(Request, _From, State) ->
+ {reply, {not_implemented, Request}, State}.
+
+-spec handle_cast(any(), state()) -> {noreply, state()}.
+handle_cast({apply_edits, Edits}, #{uri := Uri} = State) ->
+ ?LOG_DEBUG("[~p] Applying edits [uri=~p] [edits=~p]", [?MODULE, Uri, Edits]),
+ #{text := Text0, ref := Ref0} = State,
+ cancel_flush(Ref0),
+ Text = els_text:apply_edits(Text0, Edits),
+ Ref = schedule_flush(),
+ {noreply, State#{text => Text, ref => Ref}}.
+
+-spec handle_info(any(), state()) -> {noreply, state()}.
+handle_info(flush, #{uri := Uri, text := Text, pending := Pending0} = State) ->
+ ?LOG_DEBUG("[~p] Flushing [uri=~p]", [?MODULE, Uri]),
+ do_flush(Uri, Text),
+ [gen_server:reply(From, Text) || From <- Pending0],
+ {noreply, State#{pending => [], ref => undefined}};
+handle_info(_Request, State) ->
+ {noreply, State}.
+
+%%==============================================================================
+%% Internal Functions
+%%==============================================================================
+
+-spec schedule_flush() -> reference().
+schedule_flush() ->
+ erlang:send_after(?FLUSH_DELAY, self(), flush).
+
+-spec cancel_flush(undefined | reference()) -> ok.
+cancel_flush(undefined) ->
+ ok;
+cancel_flush(Ref) ->
+ erlang:cancel_timer(Ref),
+ ok.
+
+-spec do_flush(uri(), text()) -> ok.
+do_flush(Uri, Text) ->
+ {ok, Document} = els_utils:lookup_document(Uri),
+ els_indexing:deep_index(Document#{text => Text}).

apps/els_lsp/src/els_buffer_sup.erl

@@ -0,0 +1,47 @@
+%%==============================================================================
+%% Supervisor for Buffers
+%%==============================================================================
+-module(els_buffer_sup).
+
+%%==============================================================================
+%% Behaviours
+%%==============================================================================
+-behaviour(supervisor).
+
+%%==============================================================================
+%% Exports
+%%==============================================================================
+
+%% API
+-export([ start_link/0 ]).
+
+%% Supervisor Callbacks
+-export([ init/1 ]).
+
+%%==============================================================================
+%% Defines
+%%==============================================================================
+-define(SERVER, ?MODULE).
+
+%%==============================================================================
+%% API
+%%==============================================================================
+-spec start_link() -> {ok, pid()}.
+start_link() ->
+ supervisor:start_link({local, ?SERVER}, ?MODULE, []).
+
+%%==============================================================================
+%% Supervisor callbacks
+%%==============================================================================
+-spec init([]) -> {ok, {supervisor:sup_flags(), [supervisor:child_spec()]}}.
+init([]) ->
+ SupFlags = #{ strategy => simple_one_for_one
+ , intensity => 5
+ , period => 60
+ },
+ ChildSpecs = [#{ id => els_buffer_sup
+ , start => {els_buffer_server, start_link, []}
+ , restart => temporary
+ , shutdown => 5000
+ }],
+ {ok, {SupFlags, ChildSpecs}}.

apps/els_lsp/src/els_completion_provider.erl

@@ -44,8 +44,18 @@ handle_request({completion, Params}, State) ->
  }
  , <<"textDocument">> := #{<<"uri">> := Uri}
  } = Params,
- ok = els_index_buffer:flush(Uri),
- {ok, #{text := Text} = Document} = els_utils:lookup_document(Uri),
+ %% Ensure there are no pending changes.
+ {ok, Document} = els_utils:lookup_document(Uri),
+ #{buffer := Buffer, text := Text0} = Document,
+ Text = case Buffer of
+ undefined ->
+ %% This clause is only kept due to the current test suites,
+ %% where LSP clients can trigger a completion request
+ %% before a did_open
+ Text0;
+ _ ->
+ els_buffer_server:flush(Buffer)
+ end,
  Context = maps:get( <<"context">>
  , Params
  , #{ <<"triggerKind">> => ?COMPLETION_TRIGGER_KIND_INVOKED }