diff --git a/lib/ex_doc/formatter/epub/templates.ex b/lib/ex_doc/formatter/epub/templates.ex index 22dd483b8..ceb3d269e 100644 --- a/lib/ex_doc/formatter/epub/templates.ex +++ b/lib/ex_doc/formatter/epub/templates.ex @@ -13,14 +13,14 @@ defmodule ExDoc.Formatter.EPUB.Templates do defp render_doc(ast), do: ast && ExDoc.DocAST.to_string(ast) @doc """ - Generate content from the module template for a given `node` + Generate content from the module template for a given `node`. """ def module_page(config, module_node) do module_template(config, module_node) end @doc """ - Generated ID for static file + Generated ID for static file. """ def static_file_to_id(static_file) do static_file |> Path.basename() |> text_to_id() diff --git a/lib/ex_doc/formatter/html/templates.ex b/lib/ex_doc/formatter/html/templates.ex index 91c4ced07..ee7e9018a 100644 --- a/lib/ex_doc/formatter/html/templates.ex +++ b/lib/ex_doc/formatter/html/templates.ex @@ -13,7 +13,7 @@ defmodule ExDoc.Formatter.HTML.Templates do ] @doc """ - Generate content from the module template for a given `node` + Generate content from the module template for a given `node`. """ def module_page(module_node, config) do module_template(config, module_node) diff --git a/lib/ex_doc/formatter/markdown.ex b/lib/ex_doc/formatter/markdown.ex new file mode 100644 index 000000000..215b0569c --- /dev/null +++ b/lib/ex_doc/formatter/markdown.ex @@ -0,0 +1,222 @@ +defmodule ExDoc.Formatter.MARKDOWN do + @moduledoc false + + alias __MODULE__.{Templates} + alias ExDoc.Formatter + alias ExDoc.Utils + + @doc """ + Generates Markdown documentation for the given modules. + """ + @spec run([ExDoc.ModuleNode.t()], [ExDoc.ModuleNode.t()], ExDoc.Config.t()) :: String.t() + def run(project_nodes, filtered_modules, config) when is_map(config) do + Utils.unset_warned() + + config = normalize_config(config) + + build = Path.join(config.output, ".build") + output_setup(build, config) + + extras = Formatter.build_extras(config, ".md") + + project_nodes = + project_nodes + |> Formatter.render_all(filtered_modules, ".md", config, highlight_tag: "samp") + + nodes_map = %{ + modules: Formatter.filter_list(:module, project_nodes), + tasks: Formatter.filter_list(:task, project_nodes) + } + + config = %{config | extras: extras} + + all_files = + [generate_nav(config, nodes_map)] ++ + generate_extras(config) ++ + generate_list(config, nodes_map.modules) ++ + generate_list(config, nodes_map.tasks) ++ + [generate_llm_index(config, nodes_map)] + + generate_build(List.flatten(all_files), build) + config.output |> Path.join("index.md") |> Path.relative_to_cwd() + end + + defp normalize_config(config) do + output = Path.expand(config.output) + %{config | output: output} + end + + defp output_setup(build, config) do + if File.exists?(build) do + build + |> File.read!() + |> String.split("\n", trim: true) + |> Enum.map(&Path.join(config.output, &1)) + |> Enum.each(&File.rm/1) + + File.rm(build) + else + # Only remove markdown files, not HTML/EPUB files + File.mkdir_p!(config.output) + + if File.exists?(config.output) do + config.output + |> Path.join("*.md") + |> Path.wildcard() + |> Enum.each(&File.rm/1) + + llms_file = Path.join(config.output, "llms.txt") + if File.exists?(llms_file), do: File.rm(llms_file) + end + end + end + + defp generate_build(files, build) do + entries = + files + |> Enum.uniq() + |> Enum.sort() + |> Enum.map(&[&1, "\n"]) + + File.mkdir_p!(Path.dirname(build)) + File.write!(build, entries) + end + + defp normalize_output(output) do + output + |> String.replace(~r/\r\n?/, "\n") + |> String.replace(~r/\n{3,}/, "\n\n") + end + + defp generate_nav(config, nodes) do + nodes = + Map.update!(nodes, :modules, fn modules -> + modules |> Enum.chunk_by(& &1.group) |> Enum.map(&{hd(&1).group, &1}) + end) + + content = + Templates.nav_template(config, nodes) + |> normalize_output() + + filename = "index.md" + File.write("#{config.output}/#{filename}", content) + filename + end + + defp generate_extras(config) do + for {_title, extras} <- config.extras, + %{id: id, source: content} <- extras, + not is_map_key(%{id: id, source: content}, :url) do + filename = "#{id}.md" + output = "#{config.output}/#{filename}" + + if File.regular?(output) do + Utils.warn("file #{Path.relative_to_cwd(output)} already exists", []) + end + + File.write!(output, normalize_output(content)) + filename + end + end + + defp generate_list(config, nodes) do + nodes + |> Task.async_stream(&generate_module_page(&1, config), timeout: :infinity) + |> Enum.map(&elem(&1, 1)) + end + + ## Helpers + + defp generate_module_page(module_node, config) do + content = + Templates.module_page(config, module_node) + |> normalize_output() + + filename = "#{module_node.id}.md" + File.write("#{config.output}/#{filename}", content) + filename + end + + defp generate_llm_index(config, nodes_map) do + content = generate_llm_index_content(config, nodes_map) + filename = "llms.txt" + File.write("#{config.output}/#{filename}", content) + filename + end + + defp generate_llm_index_content(config, nodes_map) do + project_info = """ + # #{config.project} #{config.version} + + #{config.project} documentation index for Large Language Models. + + ## Modules + + """ + + modules_info = + nodes_map.modules + |> Enum.map(fn module_node -> + "- [#{module_node.title}](#{module_node.id}.md): #{module_node.doc |> ExDoc.DocAST.synopsis() |> extract_plain_text()}" + end) + |> Enum.join("\n") + + tasks_info = + if length(nodes_map.tasks) > 0 do + tasks_list = + nodes_map.tasks + |> Enum.map(fn task_node -> + "- [#{task_node.title}](#{task_node.id}.md): #{task_node.doc |> ExDoc.DocAST.synopsis() |> extract_plain_text()}" + end) + |> Enum.join("\n") + + "\n\n## Mix Tasks\n\n" <> tasks_list + else + "" + end + + extras_info = + if is_list(config.extras) and length(config.extras) > 0 do + extras_list = + config.extras + |> Enum.flat_map(fn + {_group, extras} when is_list(extras) -> extras + _ -> [] + end) + |> Enum.map(fn extra -> + "- [#{extra.title}](#{extra.id}.md)" + end) + |> Enum.join("\n") + + if extras_list == "" do + "" + else + "\n\n## Guides\n\n" <> extras_list + end + else + "" + end + + project_info <> modules_info <> tasks_info <> extras_info + end + + defp extract_plain_text(html) when is_binary(html) do + html + |> String.replace(~r/<[^>]*>/, "") + |> String.replace(~r/\s+/, " ") + |> String.trim() + |> case do + "" -> + "No documentation available" + + text -> + if String.length(text) > 150 do + String.slice(text, 0, 150) <> "..." + else + text + end + end + end + + defp extract_plain_text(_), do: "No documentation available" +end diff --git a/lib/ex_doc/formatter/markdown/templates.ex b/lib/ex_doc/formatter/markdown/templates.ex new file mode 100644 index 000000000..ed43583d0 --- /dev/null +++ b/lib/ex_doc/formatter/markdown/templates.ex @@ -0,0 +1,80 @@ +defmodule ExDoc.Formatter.MARKDOWN.Templates do + @moduledoc false + + require EEx + + import ExDoc.Utils, + only: [before_closing_body_tag: 2, h: 1] + + @doc """ + Generate content from the module template for a given `node`. + """ + def module_page(config, module_node) do + docs = + for group <- module_node.docs_groups do + {group.title, group.docs} + end + + module_template(config, module_node, docs) + end + + @doc """ + Formats the attribute type used to define the spec of the given `node`. + """ + def format_spec_attribute(module, node) do + module.language.format_spec_attribute(node) + end + + @doc """ + Returns the original markdown documentation from source_doc. + """ + def node_doc(%{source_doc: %{"en" => source}}) when is_binary(source), do: source + def node_doc(_), do: nil + + @doc """ + Creates a chapter which contains all the details about an individual module. + """ + EEx.function_from_file( + :def, + :module_template, + Path.expand("templates/module_template.eex", __DIR__), + [:config, :module, :docs], + trim: true + ) + + @doc """ + Creates the table of contents. + """ + EEx.function_from_file( + :def, + :nav_template, + Path.expand("templates/nav_template.eex", __DIR__), + [:config, :nodes], + trim: true + ) + + EEx.function_from_file( + :defp, + :nav_item_template, + Path.expand("templates/nav_item_template.eex", __DIR__), + [:name, :nodes], + trim: true + ) + + EEx.function_from_file( + :defp, + :nav_grouped_item_template, + Path.expand("templates/nav_grouped_item_template.eex", __DIR__), + [:nodes], + trim: true + ) + + @doc false + EEx.function_from_file( + :def, + :detail_template, + Path.expand("templates/detail_template.eex", __DIR__), + [:node, :module], + trim: true + ) +end diff --git a/lib/ex_doc/formatter/markdown/templates/detail_template.eex b/lib/ex_doc/formatter/markdown/templates/detail_template.eex new file mode 100644 index 000000000..02534ca5a --- /dev/null +++ b/lib/ex_doc/formatter/markdown/templates/detail_template.eex @@ -0,0 +1,15 @@ +# `<%= node.name %>` +<%= if node.source_url do %>[🔗](<%= node.source_url %>)<% end %> + +<%= for annotation <- node.annotations do %>*<%= annotation %>* <% end %> +<%= if deprecated = node.deprecated do %> +> This <%= node.type %> is deprecated. <%= h(deprecated) %>. +<% end %> +<%= if node.specs != [] do %> +<%= for spec <- node.specs do %> +```elixir +<%= format_spec_attribute(module, node) %> <%= spec %> +``` +<% end %> +<% end %> +<%= node_doc(node) %> diff --git a/lib/ex_doc/formatter/markdown/templates/module_template.eex b/lib/ex_doc/formatter/markdown/templates/module_template.eex new file mode 100644 index 000000000..021f63812 --- /dev/null +++ b/lib/ex_doc/formatter/markdown/templates/module_template.eex @@ -0,0 +1,12 @@ +# `<%=h module.title %>` +<%= if module.source_url do %>[🔗](<%= module.source_url %>)<% end %> + +<%= for annotation <- module.annotations do %>*<%= annotation %>* <% end %> +<%= if deprecated = module.deprecated do %> +> This <%= module.type %> is deprecated. <%=h deprecated %>. +<% end %> +<%= node_doc(module) %> +<%= for {_name, nodes} <- docs, node <- nodes do %> +<%= detail_template(node, module) %> +<% end %> +<%= before_closing_body_tag(config, :markdown) %> diff --git a/lib/ex_doc/formatter/markdown/templates/nav_grouped_item_template.eex b/lib/ex_doc/formatter/markdown/templates/nav_grouped_item_template.eex new file mode 100644 index 000000000..874ebdbfd --- /dev/null +++ b/lib/ex_doc/formatter/markdown/templates/nav_grouped_item_template.eex @@ -0,0 +1,8 @@ +<%= for {title, nodes} <- nodes do %> +<%= if title do %> +- <%=h to_string(title) %> +<% end %> +<%= for node <- nodes do %> + - [<%=h node.title %>](<%= URI.encode node.id %>.md) +<% end %> +<% end %> diff --git a/lib/ex_doc/formatter/markdown/templates/nav_item_template.eex b/lib/ex_doc/formatter/markdown/templates/nav_item_template.eex new file mode 100644 index 000000000..449c46e22 --- /dev/null +++ b/lib/ex_doc/formatter/markdown/templates/nav_item_template.eex @@ -0,0 +1,6 @@ +<%= unless Enum.empty?(nodes) do %> +- <%= name %> +<%= for node <- nodes do %> + - [<%=h node.title %>](<%= URI.encode node.id %>.md) +<% end %> +<% end %> diff --git a/lib/ex_doc/formatter/markdown/templates/nav_template.eex b/lib/ex_doc/formatter/markdown/templates/nav_template.eex new file mode 100644 index 000000000..55434a410 --- /dev/null +++ b/lib/ex_doc/formatter/markdown/templates/nav_template.eex @@ -0,0 +1,9 @@ +# <%= config.project %> v<%= config.version %> - Documentation - Table of Contents + +<%= nav_grouped_item_template config.extras %> +<%= unless Enum.empty?(nodes.modules) do %> +## Modules +<%= nav_grouped_item_template nodes.modules %> +<% end %> +<%= nav_item_template "Mix Tasks", nodes.tasks %> +<%= before_closing_body_tag(config, :markdown) %> diff --git a/lib/mix/tasks/docs.ex b/lib/mix/tasks/docs.ex index b6f03e7aa..29d571daf 100644 --- a/lib/mix/tasks/docs.ex +++ b/lib/mix/tasks/docs.ex @@ -12,9 +12,9 @@ defmodule Mix.Tasks.Docs do * `--canonical`, `-n` - Indicate the preferred URL with `rel="canonical"` link element, defaults to no canonical path - * `--formatter`, `-f` - Which formatters to use, `html` or - `epub`. This option can be given more than once. By default, - both `html` and `epub` are generated. + * `--formatter`, `-f` - Which formatters to use, `html`, + `epub`, or `markdown`. This option can be given more than once. By default, + `html`, `epub`, and `markdown` are generated. * `--language` - Specifies the language to annotate the EPUB output in valid [BCP 47](https://tools.ietf.org/html/bcp47) @@ -130,7 +130,7 @@ defmodule Mix.Tasks.Docs do against the complete module name (which includes the "Elixir." prefix for Elixir modules). If a module has `@moduledoc false`, then it is always excluded. - * `:formatters` - Formatter to use; default: ["html", "epub"], options: "html", "epub". + * `:formatters` - Formatter to use; default: ["html", "epub", "markdown"], options: "html", "epub", "markdown". * `:footer` - When false, does not render the footer on all pages, except for the required "Built with ExDoc" note. @@ -611,7 +611,12 @@ defmodule Mix.Tasks.Docs do results = for formatter <- options[:formatters] do - index = generator.(project, version, Keyword.put(options, :formatter, formatter)) + formatter_options = + options + |> Keyword.put(:formatter, formatter) + |> update_output_for_formatter(formatter) + + index = generator.(project, version, formatter_options) Mix.shell().info([:green, "View #{inspect(formatter)} docs at #{inspect(index)}"]) if cli_opts[:open] do @@ -650,13 +655,23 @@ defmodule Mix.Tasks.Docs do defp normalize_formatters(options) do formatters = case Keyword.get_values(options, :formatter) do - [] -> options[:formatters] || ["html", "epub"] + [] -> options[:formatters] || ["html", "epub", "markdown"] values -> values end Keyword.put(options, :formatters, formatters) end + defp update_output_for_formatter(options, "markdown") do + output = options[:output] || "doc" + Keyword.put(options, :output, Path.join(output, "markdown")) + end + + defp update_output_for_formatter(options, _formatter) do + output = options[:output] || "doc" + Keyword.put(options, :output, output) + end + defp get_docs_opts(config) do docs = config[:docs] diff --git a/test/ex_doc/formatter/markdown_test.exs b/test/ex_doc/formatter/markdown_test.exs new file mode 100644 index 000000000..6c437684a --- /dev/null +++ b/test/ex_doc/formatter/markdown_test.exs @@ -0,0 +1,362 @@ +defmodule ExDoc.Formatter.MarkdownTest do + use ExUnit.Case, async: false + + @moduletag :tmp_dir + + defp doc_config(%{tmp_dir: tmp_dir} = _context) do + [ + project: "Elixir", + version: "1.0.1", + formatter: "markdown", + output: tmp_dir, + source_beam: "test/tmp/beam", + skip_undefined_reference_warnings_on: ["Warnings"] + ] + end + + defp doc_config(context, config) when is_map(context) and is_list(config) do + Keyword.merge(doc_config(context), config) + end + + defp generate_docs(config) do + ExDoc.generate_docs(config[:project], config[:version], config) + end + + test "generates Markdown files in the default directory", %{tmp_dir: tmp_dir} = context do + generate_docs(doc_config(context)) + assert File.regular?(tmp_dir <> "/index.md") + assert File.regular?(tmp_dir <> "/CompiledWithDocs.md") + end + + test "generates headers for module pages", %{tmp_dir: tmp_dir} = context do + generate_docs(doc_config(context, main: "RandomError")) + + content = File.read!(tmp_dir <> "/RandomError.md") + assert content =~ ~r{^# `RandomError`}m + end + + test "generates all listing files", %{tmp_dir: tmp_dir} = context do + generate_docs(doc_config(context)) + + assert File.regular?(tmp_dir <> "/CompiledWithDocs.md") + assert File.regular?(tmp_dir <> "/CompiledWithDocs.Nested.md") + assert File.regular?(tmp_dir <> "/CustomBehaviourOne.md") + assert File.regular?(tmp_dir <> "/CustomBehaviourTwo.md") + assert File.regular?(tmp_dir <> "/RandomError.md") + assert File.regular?(tmp_dir <> "/CustomProtocol.md") + assert File.regular?(tmp_dir <> "/Mix.Tasks.TaskWithDocs.md") + end + + test "generates the index file", %{tmp_dir: tmp_dir} = context do + generate_docs(doc_config(context)) + + content = File.read!(tmp_dir <> "/index.md") + assert content =~ ~r{^# Elixir v1\.0\.1 - Documentation - Table of Contents$}m + assert content =~ ~r{## Modules} + assert content =~ ~r{- \[CompiledWithDocs\]\(CompiledWithDocs\.md\)} + assert content =~ ~r{- \[CompiledWithDocs\.Nested\]\(CompiledWithDocs\.Nested\.md\)} + end + + test "generates module with proper structure", %{tmp_dir: tmp_dir} = context do + generate_docs(doc_config(context)) + + content = File.read!(tmp_dir <> "/CompiledWithDocs.md") + + # Header + assert content =~ ~r{^# `CompiledWithDocs`}m + assert content =~ ~r{\*example_module_tag\*} + + # Moduledoc + assert content =~ ~r{moduledoc} + end + + test "generates functions correctly", %{tmp_dir: tmp_dir} = context do + generate_docs(doc_config(context)) + + content = File.read!(tmp_dir <> "/CompiledWithDocs.md") + + # Function header + assert content =~ ~r{^# `example`$}m + + # Function documentation + assert content =~ ~r{Some example} + + # Deprecated notice + assert content =~ ~r{> This function is deprecated\. Use something else instead\.} + + # Struct + assert content =~ ~r{^# `__struct__`$}m + assert content =~ ~r{Some struct} + + # Since annotation + assert content =~ ~r{^# `example_1`$}m + assert content =~ ~r{\*since 1\.3\.0\*} + + # Macro annotation + assert content =~ ~r{\*macro\*} + end + + describe "generates extras" do + test "ignores any external url extras", %{tmp_dir: tmp_dir} = context do + config = + context + |> doc_config() + |> Keyword.put(:extras, elixir: [url: "https://elixir-lang.org"]) + + generate_docs(config) + + refute File.exists?(tmp_dir <> "/elixir.md") + end + end + + test "generates LLM index file", %{tmp_dir: tmp_dir} = context do + generate_docs(doc_config(context)) + + content = File.read!(tmp_dir <> "/llms.txt") + + assert content =~ ~r{# Elixir 1\.0\.1} + assert content =~ ~r{Elixir documentation index for Large Language Models} + assert content =~ ~r{## Modules} + assert content =~ ~r{- \[CompiledWithDocs\]\(CompiledWithDocs\.md\):} + end + + describe "markdown output validation" do + test "generates proper markdown syntax", %{tmp_dir: tmp_dir} = context do + generate_docs(doc_config(context)) + content = File.read!(tmp_dir <> "/CompiledWithDocs.md") + + # Module header with backticks + assert content =~ ~r/^# `CompiledWithDocs`/m + # Function headers with backticks + assert content =~ ~r/^# `[a-z_]+`$/m + end + + test "handles complex documentation elements", %{tmp_dir: tmp_dir} = context do + generate_docs(doc_config(context)) + content = File.read!(tmp_dir <> "/CompiledWithDocs.md") + + # Function header format with backticks + assert content =~ ~r/^# `example`$/m + # Deprecated notice + assert content =~ ~r/> This function is deprecated\./ + end + + test "generates valid navigation structure", %{tmp_dir: tmp_dir} = context do + generate_docs(doc_config(context)) + content = File.read!(tmp_dir <> "/index.md") + + assert content =~ ~r/^# Elixir v1\.0\.1 - Documentation - Table of Contents$/m + assert content =~ ~r/- \[CompiledWithDocs\]\(CompiledWithDocs\.md\)/ + assert content =~ ~r/- \[CompiledWithDocs\.Nested\]\(CompiledWithDocs\.Nested\.md\)/ + assert content =~ ~r/- \[mix task_with_docs\]\(Mix\.Tasks\.TaskWithDocs\.md\)/ + refute content =~ ~r/\]\([^)]*\s[^)]*\)/ + refute content =~ ~r/\[[^\]]*\]\(\)/ + end + + test "generates proper markdown escaping", %{tmp_dir: tmp_dir} = context do + generate_docs(doc_config(context)) + content = File.read!(tmp_dir <> "/CompiledWithDocs.md") + + assert content =~ ~r/<|>|&/ || !String.contains?(content, "