Skip to content

Commit

Permalink
Merge pull request #518 from peek-travel/deps-tree-perf-improvements
Browse files Browse the repository at this point in the history 
Improve performance of algorithm to determine project deps
  • Loading branch information
@jeremyjh
jeremyjh committed Sep 16, 2023
2 parents 0173ea5 + c407d7c commit b4167c0
Showing 1 changed file with 141 additions and 74 deletions.
215 changes: 141 additions & 74 deletions lib/dialyxir/project.ex
View file
Expand Up @@ -6,6 +6,9 @@ defmodule Dialyxir.Project do
alias Dialyxir.Formatter.Short
alias Dialyxir.Formatter.Utils

# Maximum depth in the dependency tree to traverse before giving up.
@max_dep_traversal_depth 100

def plts_list(deps, include_project \\ true, exclude_core \\ false) do
elixir_apps = [:elixir]
erlang_apps = [:erts, :kernel, :stdlib, :crypto]
Expand Down Expand Up @@ -308,124 +311,136 @@ defmodule Dialyxir.Project do
defp include_deps do
method = dialyzer_config()[:plt_add_deps]

reduce_umbrella_children([], fn deps ->
deps ++
initial_acc = {
_loaded_apps = [],
_unloaded_apps = [],
_initial_load_statuses = %{}
}

{loaded_apps, _unloaded_apps, _final_load_statuses} =
reduce_umbrella_children(initial_acc, fn acc ->
case method do
false ->
[]
acc

# compatibility
true ->
warning(
"Dialyxir has deprecated plt_add_deps: true in favor of apps_direct, which includes only runtime dependencies."
)

deps_project() ++ deps_app(false)
acc
|> load_project_deps()
|> load_external_deps(recursive: false)

:project ->
warning(
"Dialyxir has deprecated plt_add_deps: :project in favor of apps_direct, which includes only runtime dependencies."
)

deps_project() ++ deps_app(false)
acc
|> load_project_deps()
|> load_external_deps(recursive: false)

:apps_direct ->
deps_app(false)
load_external_deps(acc, recursive: false)

:transitive ->
warning(
"Dialyxir has deprecated plt_add_deps: :transitive in favor of app_tree, which includes only runtime dependencies."
)

deps_transitive() ++ deps_app(true)
acc
|> load_transitive_deps()
|> load_external_deps(recursive: true)

_app_tree ->
deps_app(true)
load_external_deps(acc, recursive: true)
end
end)
end)

loaded_apps
end

defp deps_project do
Mix.Project.config()[:deps]
|> Enum.filter(&env_dep(&1))
|> Enum.map(&elem(&1, 0))
defp load_project_deps({loaded_apps, unloaded_apps, load_statuses}) do
apps =
Mix.Project.config()[:deps]
|> Enum.filter(&env_dep(&1))
|> Enum.map(&elem(&1, 0))

app_load_statuses = Map.new(apps, &{elem(&1, 0), :loaded})

update_load_statuses({loaded_apps, unloaded_apps -- apps, load_statuses}, app_load_statuses)
end

defp deps_transitive do
Mix.Project.deps_paths()
|> Map.keys()
defp load_transitive_deps({loaded_apps, unloaded_apps, load_statuses}) do
apps = Mix.Project.deps_paths() |> Map.values()
app_load_statuses = Map.new(apps, &{elem(&1, 0), :loaded})

update_load_statuses({loaded_apps, unloaded_apps -- apps, load_statuses}, app_load_statuses)
end

@spec deps_app(boolean()) :: [atom]
defp deps_app(recursive) do
defp load_external_deps({loaded_apps, _unloaded_apps, load_statuses}, opts) do
# Non-recursive traversal of 2 tries to load the app immediate deps.
traversal_depth =
case Keyword.fetch!(opts, :recursive) do
true -> @max_dep_traversal_depth
false -> 2
end

app = Mix.Project.config()[:app]
deps_app(app, recursive)
end

if System.version() |> Version.parse!() |> then(&(&1.major >= 1 and &1.minor >= 15)) do
@spec deps_app(atom(), boolean()) :: [atom()]
defp deps_app(app, recursive) do
case do_load_app(app) do
:ok ->
with_each =
if recursive do
&deps_app(&1, true)
else
fn _ -> [] end
end
# Even if already loaded, we'll need to traverse it again to get its deps.
load_statuses_w_app = Map.put(load_statuses, app, {:unloaded, :required})
traverse_deps_for_apps({loaded_apps -- [app], [app], load_statuses_w_app}, traversal_depth)
end

# Identify the optional applications which can't be loaded and thus not available
missing_apps =
Application.spec(app, :optional_applications)
|> List.wrap()
|> Enum.reject(&(do_load_app(&1) == :ok))
defp traverse_deps_for_apps({loaded_apps, [] = unloaded_deps, load_statuses}, _rem_depth),
do: {loaded_apps, unloaded_deps, load_statuses}

# Remove the optional applications which are not available from all the applications
required_apps =
Application.spec(app, :applications)
|> List.wrap()
|> Enum.reject(&(&1 in missing_apps))
defp traverse_deps_for_apps({loaded_apps, unloaded_deps, load_statuses}, 0 = _rem_depth),
do: {loaded_apps, unloaded_deps, load_statuses}

required_apps |> Stream.flat_map(&with_each.(&1)) |> Enum.concat(required_apps)
defp traverse_deps_for_apps({loaded_apps, apps_to_load, load_statuses}, rem_depth) do
initial_acc = {loaded_apps, [], load_statuses}

{:error, err} ->
error("Error loading #{app}, dependency list may be incomplete.\n #{inspect(err)}")
{updated_loaded_apps, updated_unloaded_apps, updated_load_statuses} =
Enum.reduce(apps_to_load, initial_acc, fn app, acc ->
required? = Map.fetch!(load_statuses, app) == {:unloaded, :required}
{app_load_status, app_dep_statuses} = load_app(app, required?)

[]
end
end
else
@spec deps_app(atom(), boolean()) :: [atom()]
defp deps_app(app, recursive) do
with_each =
if recursive do
&deps_app(&1, true)
else
fn _ -> [] end
end
acc
|> update_load_statuses(%{app => app_load_status})
|> update_load_statuses(app_dep_statuses)
end)

case do_load_app(app) do
:ok ->
nil
traverse_deps_for_apps(
{updated_loaded_apps, updated_unloaded_apps, updated_load_statuses},
rem_depth - 1
)
end

{:error, err} ->
error("Error loading #{app}, dependency list may be incomplete.\n #{inspect(err)}")
defp load_app(app, required?) do
case do_load_app(app) do
:ok ->
{dependencies, optional_deps} = app_dep_specs(app)

nil
end
dep_statuses =
Map.new(dependencies, fn dep ->
case dep in optional_deps do
true -> {dep, {:unloaded, :optional}}
false -> {dep, {:unloaded, :required}}
end
end)

case Application.spec(app, :applications) do
[] ->
[]
{:loaded, dep_statuses}

nil ->
[]
{:error, err} ->
if required? do
error("Error loading #{app}, dependency list may be incomplete.\n #{inspect(err)}")
end

this_apps ->
Enum.map(this_apps, with_each)
|> List.flatten()
|> Enum.concat(this_apps)
end
{{:error, err}, %{}}
end
end

Expand All @@ -443,6 +458,58 @@ defmodule Dialyxir.Project do
end
end

if System.version() |> Version.parse!() |> then(&(&1.major >= 1 and &1.minor >= 15)) do
defp app_dep_specs(app) do
# Values returned by :optional_applications are also in :applications.
dependencies = Application.spec(app, :applications) || []
optional_deps = Application.spec(app, :optional_applications) || []

{dependencies, optional_deps}
end
else
defp app_dep_specs(app) do
{Application.spec(app, :applications) || [], []}
end
end

defp update_load_statuses({loaded_apps, unloaded_apps, load_statuses}, new_statuses) do
initial_acc = {loaded_apps, unloaded_apps, load_statuses}

Enum.reduce(new_statuses, initial_acc, fn {app, new_status}, acc ->
{current_loaded_apps, current_unloaded_apps, statuses} = acc
existing_status = Map.get(statuses, app, :unset)

{new_loaded_apps, new_unloaded_apps, updated_load_statuses} =
case {existing_status, new_status} do
{:unset, {:unloaded, _} = new_status} ->
# Haven't seen this app before.
{[], [app], Map.put(statuses, app, new_status)}

{{:unloaded, :optional}, {:unloaded, :required}} ->
# A previous app had this as optional, but another one requires it.
{[], [], Map.put(statuses, app, {:unloaded, :required})}

{{:unloaded, _}, :loaded} ->
# Final state. Dependency successfully loaded.
{[app], [], Map.put(statuses, app, :loaded)}

{{:unloaded, _}, {:error, err}} ->
# Final state. Dependency failed to load.
{[], [], Map.put(statuses, app, {:error, err})}

{_prev_unloaded_or_final, _nwe_unloaded_or_final} ->
# No status change, or one that doesn't matter like final to final.
{[], [], statuses}
end

{
new_loaded_apps ++ current_loaded_apps,
new_unloaded_apps ++ current_unloaded_apps,
updated_load_statuses
}
end)
end

defp env_dep(dep) do
only_envs = dep_only(dep)
only_envs == nil || Mix.env() in List.wrap(only_envs)
Expand All @@ -452,7 +519,7 @@ defmodule Dialyxir.Project do
defp dep_only({_, _, opts}) when is_list(opts), do: opts[:only]
defp dep_only(_), do: nil

@spec reduce_umbrella_children(list(), (list() -> list())) :: list()
@spec reduce_umbrella_children(acc, (acc -> acc)) :: acc when acc: term
defp reduce_umbrella_children(acc, f) do
if Mix.Project.umbrella?() do
children = Mix.Dep.Umbrella.loaded()
Expand Down

0 comments on commit b4167c0

Please sign in to comment.