diff --git a/lib/solargraph/api_map.rb b/lib/solargraph/api_map.rb index 26b42ddb4..e33030980 100755 --- a/lib/solargraph/api_map.rb +++ b/lib/solargraph/api_map.rb @@ -118,7 +118,7 @@ def catalog bench recreate_docmap = @unresolved_requires != unresolved_requires || # @sg-ignore Unresolved call to rbs_collection_path on Solargraph::Workspace, nil workspace.rbs_collection_path != bench.workspace.rbs_collection_path || - @doc_map.uncached_gemspecs.any? + @doc_map.any_uncached? if recreate_docmap @doc_map = DocMap.new(unresolved_requires, bench.workspace, out: nil) # @todo Implement gem preferences @@ -170,16 +170,6 @@ def uncached_gemspecs doc_map.uncached_gemspecs || [] end - # @return [::Array] - def uncached_rbs_collection_gemspecs - @doc_map.uncached_rbs_collection_gemspecs - end - - # @return [::Array] - def uncached_yard_gemspecs - @doc_map.uncached_yard_gemspecs - end - # @return [Enumerable] def core_pins @@core_map.pins @@ -241,7 +231,7 @@ def self.load directory, loose_unions: true # @param rebuild [Boolean] whether to rebuild the pins even if they are cached # @return [void] def cache_all_for_doc_map! out: $stderr, rebuild: false - doc_map.cache_all!(out, rebuild: rebuild) + doc_map.cache_doc_map_gems!(out, rebuild: rebuild) end # @param gemspec [Gem::Specification] @@ -750,10 +740,10 @@ def resolve_method_aliases pins, visibility = %i[public private protected] logger.debug do "ApiMap#resolve_method_aliases(pins=#{pins.map(&:name)}, visibility=#{visibility}) => #{with_resolved_aliases.map(&:name)}" end - with_resolved_aliases + GemPins.combine_method_pins_by_path(with_resolved_aliases) end - # @return [Workspace, nil] + # @return [Workspace] def workspace doc_map.workspace end diff --git a/lib/solargraph/doc_map.rb b/lib/solargraph/doc_map.rb index 6ad366d2b..f69b87d35 100644 --- a/lib/solargraph/doc_map.rb +++ b/lib/solargraph/doc_map.rb @@ -5,123 +5,83 @@ require 'open3' module Solargraph - # A collection of pins generated from required gems. + # A collection of pins generated from specific 'require' statements + # in code. Multiple can be created per workspace, to represent the + # pins available in different files based on their particular + # 'require' lines. # class DocMap include Logging - # @return [Array] - attr_reader :requires - alias required requires - - # @return [Array] - attr_reader :preferences - - # @return [Array] - attr_reader :pins - - # @return [Array] - def uncached_gemspecs - uncached_yard_gemspecs.concat(uncached_rbs_collection_gemspecs) - .sort - .uniq { |gemspec| "#{gemspec.name}:#{gemspec.version}" } - end - - # @return [Array] - attr_reader :uncached_yard_gemspecs - - # @return [Array] - attr_reader :uncached_rbs_collection_gemspecs - - # @return [String, nil] - attr_reader :rbs_collection_path - - # @return [String, nil] - attr_reader :rbs_collection_config_path - - # @return [Workspace, nil] + # @return [Workspace] attr_reader :workspace - # @return [Environ] - attr_reader :environ - # @param requires [Array] # @param workspace [Workspace, nil] - # @param [Object] out + # @param out [IO, nil] output stream for logging def initialize requires, workspace, out: $stderr - @requires = requires.compact + @provided_requires = requires.compact @workspace = workspace - @rbs_collection_path = workspace&.rbs_collection_path - @rbs_collection_config_path = workspace&.rbs_collection_config_path - @environ = Convention.for_global(self) - @requires.concat @environ.requires if @environ - load_serialized_gem_pins - pins.concat @environ.pins @out = out end - # @param out [IO, StringIO, nil] - # @return [void] - # @param [Boolean] rebuild - def cache_all! out, rebuild: false - # if we log at debug level: - if logger.info? - gem_desc = uncached_gemspecs.map { |gemspec| "#{gemspec.name}:#{gemspec.version}" }.join(', ') - logger.info "Caching pins for gems: #{gem_desc}" unless uncached_gemspecs.empty? - end - logger.debug { "Caching for YARD: #{uncached_yard_gemspecs.map(&:name)}" } - logger.debug { "Caching for RBS collection: #{uncached_rbs_collection_gemspecs.map(&:name)}" } - load_serialized_gem_pins - uncached_gemspecs.each do |gemspec| - cache(gemspec, rebuild: rebuild, out: out) + # @return [Array] + def requires + @requires ||= @provided_requires + (workspace.global_environ&.requires || []) + end + alias required requires + + # @sg-ignore flow sensitive typing needs to understand reassignment + # @return [Array] + def uncached_gemspecs + if @uncached_gemspecs.nil? + @uncached_gemspecs = [] + pins # force lazy-loaded pin lookup end - load_serialized_gem_pins - @uncached_rbs_collection_gemspecs = [] - @uncached_yard_gemspecs = [] + @uncached_gemspecs end - # @param gemspec [Gem::Specification] - # @param out [IO, StringIO, nil] - # @return [void] - def cache_yard_pins gemspec, out - pins = GemPins.build_yard_pins(yard_plugins, gemspec) - PinCache.serialize_yard_gem(gemspec, pins) - logger.info { "Cached #{pins.length} YARD pins for gem #{gemspec.name}:#{gemspec.version}" } unless pins.empty? + # @return [Array] + def pins + @pins ||= load_serialized_gem_pins + (workspace.global_environ&.pins || []) end - # @param gemspec [Gem::Specification] - # @param out [IO, StringIO, nil] # @return [void] - def cache_rbs_collection_pins gemspec, out - rbs_map = RbsMap.from_gemspec(gemspec, rbs_collection_path, rbs_collection_config_path) - pins = rbs_map.pins - rbs_version_cache_key = rbs_map.cache_key - # cache pins even if result is zero, so we don't retry building pins - pins ||= [] - PinCache.serialize_rbs_collection_gem(gemspec, rbs_version_cache_key, pins) - logger.info { "Cached #{pins.length} RBS collection pins for gem #{gemspec.name} #{gemspec.version} with cache_key #{rbs_version_cache_key.inspect}" unless pins.empty? } + def reset_pins! + @uncached_gemspecs = nil + @pins = nil end - # @param gemspec [Gem::Specification] + # @return [Solargraph::PinCache] + def pin_cache + @pin_cache ||= workspace.fresh_pincache + end + + def any_uncached? + uncached_gemspecs.any? + end + + # Cache all pins needed for the sources in this doc_map + # @param out [StringIO, IO, nil] output stream for logging # @param rebuild [Boolean] whether to rebuild the pins even if they are cached - # @param out [IO, StringIO, nil] output stream for logging # @return [void] - def cache gemspec, rebuild: false, out: nil - build_yard = uncached_yard_gemspecs.include?(gemspec) || rebuild - build_rbs_collection = uncached_rbs_collection_gemspecs.include?(gemspec) || rebuild - if build_yard || build_rbs_collection - type = [] - type << 'YARD' if build_yard - type << 'RBS collection' if build_rbs_collection - out&.puts("Caching #{type.join(' and ')} pins for gem #{gemspec.name}:#{gemspec.version}") + def cache_doc_map_gems! out, rebuild: false + unless uncached_gemspecs.empty? + logger.info do + gem_desc = uncached_gemspecs.map { |gemspec| "#{gemspec.name}:#{gemspec.version}" }.join(', ') + "Caching pins for gems: #{gem_desc}" + end end - cache_yard_pins(gemspec, out) if build_yard - cache_rbs_collection_pins(gemspec, out) if build_rbs_collection - end - - # @return [Array] - def gemspecs - @gemspecs ||= required_gems_map.values.compact.flatten + time = Benchmark.measure do + uncached_gemspecs.each do |gemspec| + cache(gemspec, rebuild: rebuild, out: out) + end + end + milliseconds = (time.real * 1000).round + if (milliseconds > 500) && uncached_gemspecs.any? && out && uncached_gemspecs.any? + out.puts "Built #{uncached_gemspecs.length} gems in #{milliseconds} ms" + end + reset_pins! end # @return [Array] @@ -129,311 +89,112 @@ def unresolved_requires @unresolved_requires ||= required_gems_map.select { |_, gemspecs| gemspecs.nil? }.keys end - # @return [Hash{Array(String, String) => Array}] Indexed by gemspec name and version - def self.all_yard_gems_in_memory - @all_yard_gems_in_memory ||= {} - end - - # @return [Hash{String => Hash{Array(String, String) => Array}}] stored by RBS collection path - def self.all_rbs_collection_gems_in_memory - @all_rbs_collection_gems_in_memory ||= {} - end - - # @return [Hash{Array(String, String) => Array}] Indexed by gemspec name and version - def yard_pins_in_memory - self.class.all_yard_gems_in_memory - end - - # @return [Hash{Array(String, String) => Array}] Indexed by gemspec name and version - def rbs_collection_pins_in_memory - # @sg-ignore rbs_collection_path is String | nil but used as hash key - self.class.all_rbs_collection_gems_in_memory[rbs_collection_path] ||= {} - end - - # @return [Hash{Array(String, String) => Array}] Indexed by gemspec name and version - def self.all_combined_pins_in_memory - @all_combined_pins_in_memory ||= {} + # @return [Array] + # @param out [IO, nil] + def dependencies out: $stderr + @dependencies ||= + begin + gem_deps = gemspecs + .flat_map { |spec| workspace.fetch_dependencies(spec, out: out) } + .uniq(&:name) + stdlib_deps = gemspecs + .flat_map { |spec| workspace.stdlib_dependencies(spec.name) } + .flat_map { |dep_name| workspace.resolve_require(dep_name) } + .compact + existing_gems = gemspecs.map(&:name) + (gem_deps + stdlib_deps).reject { |gemspec| existing_gems.include? gemspec.name } + end end - # @todo this should also include an index by the hash of the RBS collection - # @return [Hash{Array(String, String) => Array}] Indexed by gemspec name and version - def combined_pins_in_memory - self.class.all_combined_pins_in_memory + # Cache gem documentation if needed for this doc_map + # + # @param gemspec [Gem::Specification] + # @param rebuild [Boolean] whether to rebuild the pins even if they are cached + # @param out [StringIO, IO, nil] output stream for logging + # + # @return [void] + def cache gemspec, rebuild: false, out: nil + pin_cache.cache_gem(gemspec: gemspec, + rebuild: rebuild, + out: out) end - # @return [Array] - def yard_plugins - @environ.yard_plugins - end + private - # @return [Set] - def dependencies - @dependencies ||= (gemspecs.flat_map { |spec| fetch_dependencies(spec) } - gemspecs).to_set + # @return [Array] + def gemspecs + @gemspecs ||= required_gems_map.values.compact.flatten end - private - - # @return [void] - def load_serialized_gem_pins - @pins = [] - @uncached_yard_gemspecs = [] - @uncached_rbs_collection_gemspecs = [] + # @param out [IO, nil] + # @return [Array] + def load_serialized_gem_pins out: @out + serialized_pins = [] with_gemspecs, without_gemspecs = required_gems_map.partition { |_, v| v } # @type [Array] - paths = without_gemspecs.to_h.keys + missing_paths = without_gemspecs.to_h.keys # @type [Array] - gemspecs = with_gemspecs.to_h.values.flatten.compact + dependencies.to_a - - paths.each do |path| - deserialize_stdlib_rbs_map path + gemspecs = with_gemspecs.to_h.values.flatten.compact + dependencies(out: out).to_a + + # if we are type checking a gem project, we should not include + # pins from rbs or yard from that gem here - we use our own + # parser for those pins + + # @param gemspec [Gem::Specification, Bundler::LazySpecification, Bundler::StubSpecification] + gemspecs.reject! do |gemspec| + gemspec.respond_to?(:source) && + gemspec.source.instance_of?(Bundler::Source::Gemspec) && + gemspec.source.respond_to?(:path) && + gemspec.source.path == Pathname.new('.') + end + + missing_paths.each do |path| + # this will load from disk if needed; no need to manage + # uncached_gemspecs to trigger that later + stdlib_name_guess = path.split('/').first + + # try to resolve the stdlib name + # @type [Array] + deps = workspace.stdlib_dependencies(stdlib_name_guess) || [] + [stdlib_name_guess, *deps].compact.each do |potential_stdlib_name| + # @sg-ignore Need to support splatting in literal array + rbs_pins = pin_cache.cache_stdlib_rbs_map potential_stdlib_name + serialized_pins.concat rbs_pins if rbs_pins + end end - logger.debug { 'DocMap#load_serialized_gem_pins: Combining pins...' } + serialized_pins.length time = Benchmark.measure do gemspecs.each do |gemspec| - pins = deserialize_combined_pin_cache gemspec - @pins.concat pins if pins + gemspec_pins = pin_cache.deserialize_combined_pin_cache gemspec + if gemspec_pins + # deserialize_combined_pin_cache may have answered with a + # fallback - RbsMap#fallback_pins - before the real combined + # cache exists, so confirm that's not what happened rather + # than inferring "cached" from "got pins back". + serialized_pins.concat gemspec_pins + uncached_gemspecs << gemspec unless pin_cache.cached?(gemspec) + else + uncached_gemspecs << gemspec + end end end - logger.info { "DocMap#load_serialized_gem_pins: Loaded and processed serialized pins together in #{time.real} seconds" } - @uncached_yard_gemspecs.uniq! - @uncached_rbs_collection_gemspecs.uniq! - nil + serialized_pins.length + milliseconds = (time.real * 1000).round + if (milliseconds > 500) && out && gemspecs.any? + out.puts "Deserialized #{serialized_pins.length} gem pins from #{PinCache.base_dir} in #{milliseconds} ms" + end + uncached_gemspecs.uniq! { |gemspec| "#{gemspec.name}:#{gemspec.version}" } + serialized_pins end # @return [Hash{String => Array}] def required_gems_map - @required_gems_map ||= requires.to_h { |path| [path, resolve_path_to_gemspecs(path)] } - end - - # @return [Hash{String => Gem::Specification}] - def preference_map - @preference_map ||= preferences.to_h { |gemspec| [gemspec.name, gemspec] } - end - - # @param gemspec [Gem::Specification] - # @return [Array, nil] - def deserialize_yard_pin_cache gemspec - if yard_pins_in_memory.key?([gemspec.name, gemspec.version]) - return yard_pins_in_memory[[gemspec.name, gemspec.version]] - end - - cached = PinCache.deserialize_yard_gem(gemspec) - if cached - logger.info { "Loaded #{cached.length} cached YARD pins from #{gemspec.name}:#{gemspec.version}" } - yard_pins_in_memory[[gemspec.name, gemspec.version]] = cached - cached - else - logger.debug "No YARD pin cache for #{gemspec.name}:#{gemspec.version}" - @uncached_yard_gemspecs.push gemspec - nil - end - end - - # @param gemspec [Gem::Specification] - # @return [void] - def deserialize_combined_pin_cache gemspec - unless combined_pins_in_memory[[gemspec.name, gemspec.version]].nil? - return combined_pins_in_memory[[gemspec.name, gemspec.version]] - end - - rbs_map = RbsMap.from_gemspec(gemspec, rbs_collection_path, rbs_collection_config_path) - rbs_version_cache_key = rbs_map.cache_key - - cached = PinCache.deserialize_combined_gem(gemspec, rbs_version_cache_key) - if cached - logger.info { "Loaded #{cached.length} cached YARD pins from #{gemspec.name}:#{gemspec.version}" } - combined_pins_in_memory[[gemspec.name, gemspec.version]] = cached - return combined_pins_in_memory[[gemspec.name, gemspec.version]] - end - - rbs_collection_pins = deserialize_rbs_collection_cache gemspec, rbs_version_cache_key - - yard_pins = deserialize_yard_pin_cache gemspec - - if !rbs_collection_pins.nil? && !yard_pins.nil? - logger.debug { "Combining pins for #{gemspec.name}:#{gemspec.version}" } - combined_pins = GemPins.combine(yard_pins, rbs_collection_pins) - PinCache.serialize_combined_gem(gemspec, rbs_version_cache_key, combined_pins) - combined_pins_in_memory[[gemspec.name, gemspec.version]] = combined_pins - logger.info { "Generated #{combined_pins_in_memory[[gemspec.name, gemspec.version]].length} combined pins for #{gemspec.name} #{gemspec.version}" } - return combined_pins - end - - if !yard_pins.nil? - logger.debug { "Using only YARD pins for #{gemspec.name}:#{gemspec.version}" } - combined_pins_in_memory[[gemspec.name, gemspec.version]] = yard_pins - combined_pins_in_memory[[gemspec.name, gemspec.version]] - elsif !rbs_collection_pins.nil? - logger.debug { "Using only RBS collection pins for #{gemspec.name}:#{gemspec.version}" } - combined_pins_in_memory[[gemspec.name, gemspec.version]] = rbs_collection_pins - combined_pins_in_memory[[gemspec.name, gemspec.version]] - else - logger.debug { "Pins not yet cached for #{gemspec.name}:#{gemspec.version}" } - nil - end - end - - # @param path [String] require path that might be in the RBS stdlib collection - # @return [void] - def deserialize_stdlib_rbs_map path - map = RbsMap::StdlibMap.load(path) - if map.resolved? - logger.debug { "Loading stdlib pins for #{path}" } - @pins.concat map.pins - logger.debug { "Loaded #{map.pins.length} stdlib pins for #{path}" } - map.pins - else - # @todo Temporarily ignoring unresolved `require 'set'` - logger.debug { "Require path #{path} could not be resolved in RBS" } unless path == 'set' - nil - end - end - - # @param gemspec [Gem::Specification] - # @param rbs_version_cache_key [String] - # @return [Array, nil] - def deserialize_rbs_collection_cache gemspec, rbs_version_cache_key - return if rbs_collection_pins_in_memory.key?([gemspec, rbs_version_cache_key]) - cached = PinCache.deserialize_rbs_collection_gem(gemspec, rbs_version_cache_key) - if cached - logger.info { "Loaded #{cached.length} pins from RBS collection cache for #{gemspec.name}:#{gemspec.version}" } unless cached.empty? - rbs_collection_pins_in_memory[[gemspec, rbs_version_cache_key]] = cached - cached - else - logger.debug "No RBS collection pin cache for #{gemspec.name} #{gemspec.version}" - @uncached_rbs_collection_gemspecs.push gemspec - nil - end - end - - # @param path [String] - # @return [::Array, nil] - def resolve_path_to_gemspecs path - return nil if path.empty? - return gemspecs_required_from_bundler if path == 'bundler/require' - - # @type [Gem::Specification, nil] - gemspec = Gem::Specification.find_by_path(path) - if gemspec.nil? - gem_name_guess = path.split('/').first - return nil if gem_name_guess.to_s.empty? - begin - # this can happen when the gem is included via a local path in - # a Gemfile; Gem doesn't try to index the paths in that case. - # - # See if we can make a good guess: - gemspec = Gem::Specification.find_by_name(gem_name_guess) - rescue Gem::MissingSpecError - logger.debug { "Require path #{path} could not be resolved to a gem via find_by_path or guess of #{gem_name_guess}" } - [] - end - end - return nil if gemspec.nil? - [gemspec_or_preference(gemspec)] - end - - # @param gemspec [Gem::Specification] - # @return [Gem::Specification] - def gemspec_or_preference gemspec - # :nocov: dormant feature - return gemspec unless preference_map.key?(gemspec.name) - return gemspec if gemspec.version == preference_map[gemspec.name].version - - change_gemspec_version gemspec, preference_map[gemspec.name].version - # :nocov: - end - - # @param gemspec [Gem::Specification] - # @param version [Gem::Version, String] - # @return [Gem::Specification] - def change_gemspec_version gemspec, version - Gem::Specification.find_by_name(gemspec.name, "= #{version}") - rescue Gem::MissingSpecError - Solargraph.logger.info "Gem #{gemspec.name} version #{version} not found. Using #{gemspec.version} instead" - gemspec - end - - # @param gemspec [Gem::Specification] - # @return [Array] - def fetch_dependencies gemspec - # @param spec [Gem::Dependency] - # @param deps [Set] - only_runtime_dependencies(gemspec).each_with_object(Set.new) do |spec, deps| - Solargraph.logger.info "Adding #{spec.name} dependency for #{gemspec.name}" - dep = Gem.loaded_specs[spec.name] - # @todo is next line necessary? - dep ||= Gem::Specification.find_by_name(spec.name, spec.requirement) - deps.merge fetch_dependencies(dep) if deps.add?(dep) - rescue Gem::MissingSpecError - Solargraph.logger.warn "Gem dependency #{spec.name} for #{gemspec.name} not found in RubyGems." - end.to_a - end - - # @param gemspec [Gem::Specification] - # @return [Array] - def only_runtime_dependencies gemspec - gemspec.dependencies - gemspec.development_dependencies + @required_gems_map ||= requires.to_h { |path| [path, workspace.resolve_require(path)] } end def inspect self.class.inspect end - - # @return [Array, nil] - def gemspecs_required_from_bundler - # @todo Handle projects with custom Bundler/Gemfile setups - return unless workspace&.gemfile? - - # @sg-ignore workspace is checked for nil above - if workspace.gemfile? && Bundler.definition&.lockfile&.to_s&.start_with?(workspace.directory) # rubocop:disable Style/SafeNavigationChainLength - # Find only the gems bundler is now using - Bundler.definition.locked_gems.specs.flat_map do |lazy_spec| - logger.info "Handling #{lazy_spec.name}:#{lazy_spec.version}" - [Gem::Specification.find_by_name(lazy_spec.name, lazy_spec.version)] - rescue Gem::MissingSpecError => e - logger.info("Could not find #{lazy_spec.name}:#{lazy_spec.version} with find_by_name, falling back to guess") - # can happen in local filesystem references - specs = resolve_path_to_gemspecs lazy_spec.name - logger.warn "Gem #{lazy_spec.name} #{lazy_spec.version} from bundle not found: #{e}" if specs.nil? - next specs - end.compact - else - logger.info 'Fetching gemspecs required from Bundler (bundler/require)' - gemspecs_required_from_external_bundle - end - end - - # @return [Array] - def gemspecs_required_from_external_bundle - logger.info 'Fetching gemspecs required from external bundle' - return [] unless workspace&.directory - - Solargraph.with_clean_env do - cmd = [ - 'ruby', '-e', - # @sg-ignore return above ensures workspace.directory is not nil - "require 'bundler'; require 'json'; Dir.chdir('#{workspace.directory}') { puts Bundler.definition.locked_gems.specs.map { |spec| [spec.name, spec.version] }.to_h.to_json }" - ] - o, e, s = Open3.capture3(*cmd) - if s.success? - Solargraph.logger.debug "External bundle: #{o}" - hash = o && !o.empty? ? JSON.parse(o.split("\n").last) : {} - hash.flat_map do |name, version| - Gem::Specification.find_by_name(name, version) - rescue Gem::MissingSpecError => e - logger.info("Could not find #{name}:#{version} with find_by_name, falling back to guess") - # can happen in local filesystem references - specs = resolve_path_to_gemspecs name - logger.warn "Gem #{name} #{version} from bundle not found: #{e}" if specs.nil? - next specs - end.compact - else - # @sg-ignore return above ensures workspace.directory is not nil - Solargraph.logger.warn "Failed to load gems from bundle at #{workspace.directory}: #{e}" - [] - end - end - end end end diff --git a/lib/solargraph/gem_pins.rb b/lib/solargraph/gem_pins.rb index d9e731d72..8e13d6d87 100644 --- a/lib/solargraph/gem_pins.rb +++ b/lib/solargraph/gem_pins.rb @@ -11,6 +11,17 @@ class << self include Logging end + # @param pins [Array] + # @return [Array] + def self.combine_method_pins_by_path pins + method_pins, alias_pins = pins.partition { |pin| pin.instance_of?(Pin::Method) } + by_path = method_pins.group_by(&:path) + by_path.transform_values! do |pins| + GemPins.combine_method_pins(*pins) + end + by_path.values + alias_pins + end + # @param pins [Array] # @return [Pin::Method, nil] def self.combine_method_pins(*pins) @@ -32,36 +43,30 @@ def self.combine_method_pins(*pins) out end - # @param yard_plugins [Array] The names of YARD plugins to use. - # @param gemspec [Gem::Specification] - # @return [Array] - def self.build_yard_pins yard_plugins, gemspec - Yardoc.cache(yard_plugins, gemspec) unless Yardoc.cached?(gemspec) - return [] unless Yardoc.cached?(gemspec) - yardoc = Yardoc.load!(gemspec) - YardMap::Mapper.new(yardoc, gemspec).map - end - - # @param yard_pins [Array] - # @param rbs_pins [Array] + # @param yard_pins [Array] + # @param rbs_pins [Array] # - # @return [Array] + # @return [Array] def self.combine yard_pins, rbs_pins in_yard = Set.new - rbs_api_map = Solargraph::ApiMap.new(pins: rbs_pins) + rbs_store = Solargraph::ApiMap::Store.new(rbs_pins) combined = yard_pins.map do |yard_pin| in_yard.add yard_pin.path - rbs_pin = rbs_api_map.get_path_pins(yard_pin.path).filter { |pin| pin.is_a? Pin::Method }.first - next yard_pin unless rbs_pin && yard_pin.instance_of?(Pin::Method) + rbs_pin = rbs_store.get_path_pins(yard_pin.path).filter { |pin| pin.is_a? Pin::Method }.first - unless rbs_pin - # @sg-ignore https://github.com/castwide/solargraph/pull/1114 - logger.debug { "GemPins.combine: No rbs pin for #{yard_pin.path} - using YARD's '#{yard_pin.inspect} (return_type=#{yard_pin.return_type}; signatures=#{yard_pin.signatures})" } - next yard_pin - end + next yard_pin unless rbs_pin && yard_pin.is_a?(Pin::Method) + # at this point both yard_pins and rbs_pins are methods or + # method aliases. if not plain methods, prefer the YARD one + next yard_pin if rbs_pin.class != Pin::Method + + next rbs_pin if yard_pin.class != Pin::Method + + # both are method pins out = combine_method_pins(rbs_pin, yard_pin) - logger.debug { "GemPins.combine: Combining yard.path=#{yard_pin.path} - rbs=#{rbs_pin.inspect} with yard=#{yard_pin.inspect} into #{out}" } + logger.debug do + "GemPins.combine: Combining yard.path=#{yard_pin.path} - rbs=#{rbs_pin.inspect} with yard=#{yard_pin.inspect} into #{out}" + end out end in_rbs_only = rbs_pins.select do |pin| diff --git a/lib/solargraph/library.rb b/lib/solargraph/library.rb index 4f03fb862..baca20481 100644 --- a/lib/solargraph/library.rb +++ b/lib/solargraph/library.rb @@ -1,9 +1,16 @@ # frozen_string_literal: true +require 'rubygems' require 'pathname' require 'observer' require 'open3' +# @!parse +# class ::Gem::Specification +# # @return [String] +# def name; end +# end + module Solargraph # A Library handles coordination between a Workspace and an ApiMap. # @@ -33,6 +40,7 @@ def initialize workspace = Solargraph::Workspace.new, name = nil # @type [Source, nil] @current = nil @sync_count = 0 + @cache_progress = nil end def inspect @@ -265,11 +273,13 @@ def references_from filename, line, column, strip: false, only: false referenced&.path == pin.path end if pin.path == 'Class#new' + # @todo flow sensitive typing should allow shadowing of Kernel#caller caller = cursor.chain.base.infer(api_map, clip.send(:closure), clip.locals).first if caller.defined? found.select! do |loc| clip = api_map.clip_at(loc.filename, loc.range.start) other = clip.send(:cursor).chain.base.infer(api_map, clip.send(:closure), clip.locals).first + # @todo flow sensitive typing should allow shadowing of Kernel#caller caller == other end else @@ -283,9 +293,7 @@ def references_from filename, line, column, strip: false, only: false Solargraph::Location.new(loc.filename, Solargraph::Range.from_to(loc.range.start.line, loc.range.start.column + match[0].length, loc.range.ending.line, loc.range.ending.column)) end end - result.concat(found.sort do |a, b| - a.range.start.line <=> b.range.start.line - end) + result.concat(found.sort { |a, b| a.range.start.line <=> b.range.start.line }) end result.uniq end @@ -310,9 +318,7 @@ def locate_ref location return nil if pin.nil? # @param full [String] return_if_match = proc do |full| - if source_map_hash.key?(full) - return Location.new(full, Solargraph::Range.from_to(0, 0, 0, 0)) - end + return Location.new(full, Solargraph::Range.from_to(0, 0, 0, 0)) if source_map_hash.key?(full) end workspace.require_paths.each do |path| full = File.join path, pin.name @@ -480,6 +486,7 @@ def mapped? # @return [SourceMap, Boolean] def next_map return false if mapped? + # @sg-ignore Need to add nil check here src = workspace.sources.find { |s| !source_map_hash.key?(s.filename) } if src Logging.logger.debug "Mapping #{src.filename}" @@ -515,6 +522,11 @@ def external_requires private + # @return [PinCache] + def pin_cache + workspace.pin_cache + end + # @return [Hash{String => Array}] def source_map_external_require_hash @source_map_external_require_hash ||= {} @@ -576,6 +588,7 @@ def maybe_map source return unless source # @sg-ignore Wrong argument type for Solargraph::Workspace#has_file?: filename expected String, received String, nil return unless @current == source || workspace.has_file?(source.filename) + # @sg-ignore Need to add nil check here if source_map_hash.key?(source.filename) new_map = Solargraph::SourceMap.map(source) # @sg-ignore OK if source.filename is nil @@ -600,7 +613,7 @@ def cache_next_gemspec pending = api_map.uncached_gemspecs.length - cache_errors.length - 1 - if Yardoc.processing?(spec) + if pin_cache.yardoc_processing?(spec) logger.info "Enqueuing cache of #{spec.name} #{spec.version} (already being processed)" queued_gemspec_cache.push(spec) return if pending - queued_gemspec_cache.length < 1 @@ -615,7 +628,10 @@ def cache_next_gemspec logger.info "Caching #{spec.name} #{spec.version}" Thread.new do report_cache_progress spec.name, pending - _o, e, s = Open3.capture3(workspace.command_path, 'cache', spec.name, spec.version.to_s) + kwargs = {} + kwargs[:chdir] = workspace.directory.to_s if workspace.directory && !workspace.directory.empty? + _o, e, s = Open3.capture3(workspace.command_path, 'cache', spec.name, spec.version.to_s, + **kwargs) if s.success? logger.info "Cached #{spec.name} #{spec.version}" else @@ -632,8 +648,7 @@ def cache_next_gemspec # @return [Array] def cacheable_specs - cacheable = api_map.uncached_yard_gemspecs + - api_map.uncached_rbs_collection_gemspecs - + cacheable = api_map.uncached_gemspecs + queued_gemspec_cache - cache_errors.to_a return cacheable unless cacheable.empty? @@ -696,8 +711,7 @@ def sync_catalog source_map_hash.each_value { |map| find_external_requires(map) } api_map.catalog bench logger.info "Catalog complete (#{api_map.source_maps.length} files, #{api_map.pins.length} pins)" - logger.info "#{api_map.uncached_yard_gemspecs.length} uncached YARD gemspecs" - logger.info "#{api_map.uncached_rbs_collection_gemspecs.length} uncached RBS collection gemspecs" + logger.info "#{api_map.uncached_gemspecs.length} uncached gemspecs" cache_next_gemspec @sync_count = 0 end diff --git a/lib/solargraph/pin/callable.rb b/lib/solargraph/pin/callable.rb index ed87b79e4..60a68af3d 100644 --- a/lib/solargraph/pin/callable.rb +++ b/lib/solargraph/pin/callable.rb @@ -108,11 +108,10 @@ def arity [generics, blockless_parameters.map(&:arity_decl), block&.arity] end - # e.g., [["T"], "1", "?3", "foo:5"] - parameter arity - # declarations, including the number of unique types in each - # parameter. Used to determine whether combining two - # signatures has lost useful information mapping specific - # parameter types to specific return types. + # e.g., [["T"], "::String", "?::Integer", "foo:::Symbol"] - parameter + # arity declarations carrying the rooted types of each parameter. + # Used to tell whether combining two signatures loses the mapping + # from specific parameter types to specific return types. # # @return [Array] def type_arity diff --git a/lib/solargraph/pin/method.rb b/lib/solargraph/pin/method.rb index 81cc94d28..a71cd664b 100644 --- a/lib/solargraph/pin/method.rb +++ b/lib/solargraph/pin/method.rb @@ -75,7 +75,7 @@ def combine_with other, attrs = {} # @param other [Pin::Method] def == other - super && other.node == node + super && other.node == node && other.signatures == signatures end def transform_types &transform diff --git a/lib/solargraph/pin/parameter.rb b/lib/solargraph/pin/parameter.rb index ba20976ec..fff78430e 100644 --- a/lib/solargraph/pin/parameter.rb +++ b/lib/solargraph/pin/parameter.rb @@ -95,7 +95,7 @@ def arity_decl # @return [String] def type_arity_decl - arity_decl + return_type.items.count.to_s + arity_decl + return_type.rooted_tags end def arg? diff --git a/lib/solargraph/pin_cache.rb b/lib/solargraph/pin_cache.rb index 803170764..6a1ee3316 100644 --- a/lib/solargraph/pin_cache.rb +++ b/lib/solargraph/pin_cache.rb @@ -1,13 +1,454 @@ -require 'yard-activesupport-concern' +# frozen_string_literal: true + require 'fileutils' -require 'pathname' # @todo Required by RBS but not loaded in some use cases require 'rbs' +require 'rubygems' module Solargraph - module PinCache + class PinCache + include Logging + + attr_reader :directory, :rbs_collection_path, :rbs_collection_config_path, :yard_plugins + + # @param rbs_collection_path [String, nil] + # @param rbs_collection_config_path [String, nil] + # @param directory [String, nil] + # @param yard_plugins [Array] + def initialize rbs_collection_path:, rbs_collection_config_path:, + directory:, + yard_plugins: + @rbs_collection_path = rbs_collection_path + @rbs_collection_config_path = rbs_collection_config_path + @directory = directory + @yard_plugins = yard_plugins + end + + # @param gemspec [Gem::Specification, Bundler::LazySpecification] + def cached? gemspec + rbs_version_cache_key = lookup_rbs_version_cache_key(gemspec) + combined_gem?(gemspec, rbs_version_cache_key) + end + + # @param gemspec [Gem::Specification] + # @param rebuild [Boolean] whether to rebuild the cache regardless of whether it already exists + # @param out [StringIO, IO, nil] output stream for logging + # @return [void] + def cache_gem gemspec:, rebuild: false, out: nil + rbs_version_cache_key = lookup_rbs_version_cache_key(gemspec) + + build_yard, build_rbs_collection, build_combined = + calculate_build_needs(gemspec, + rebuild: rebuild, + rbs_version_cache_key: rbs_version_cache_key) + + return unless build_yard || build_rbs_collection || build_combined + + build_combine_and_cache(gemspec, + rbs_version_cache_key, + build_yard: build_yard, + build_rbs_collection: build_rbs_collection, + build_combined: build_combined, + out: out) + end + + # @param gemspec [Gem::Specification, Bundler::LazySpecification] + # @param rbs_version_cache_key [String, nil] + def suppress_yard_cache? gemspec, rbs_version_cache_key + if gemspec.name == 'parser' && rbs_version_cache_key != RbsMap::CACHE_KEY_UNRESOLVED + # parser takes forever to build YARD pins, but has excellent RBS collection pins + return true + end + false + end + + # @param out [StringIO, IO, nil] output stream for logging + # @param rebuild [Boolean] build pins regardless of whether we + # have cached them already + # + # @return [void] + def cache_all_stdlibs rebuild: false, out: $stderr + possible_stdlibs.each do |stdlib| + RbsMap::StdlibMap.new(stdlib, rebuild: rebuild, out: out) + end + end + + # @param path [String] require path that might be in the RBS stdlib collection + # @return [void] + def cache_stdlib_rbs_map path + # these are held in memory in RbsMap::StdlibMap + map = RbsMap::StdlibMap.load(path) + if map.resolved? + logger.debug { "Loading stdlib pins for #{path}" } + pins = map.pins + logger.debug { "Loaded #{pins.length} stdlib pins for #{path}" } + pins + else + # @todo Temporarily ignoring unresolved `require 'set'` + logger.debug { "Require path #{path} could not be resolved in RBS" } unless path == 'set' + nil + end + end + + # @param gemspec [Gem::Specification, Bundler::LazySpecification] + # + # @return [String] + def lookup_rbs_version_cache_key gemspec + rbs_map = RbsMap.from_gemspec(gemspec, rbs_collection_path, rbs_collection_config_path) + rbs_map.cache_key + end + + # @param gemspec [Gem::Specification, Bundler::LazySpecification] + # @param rbs_version_cache_key [String, nil] + # @param yard_pins [Array] + # @param rbs_collection_pins [Array] + # @return [void] + def cache_combined_pins gemspec, rbs_version_cache_key, yard_pins, rbs_collection_pins + combined_pins = GemPins.combine(yard_pins, rbs_collection_pins) + serialize_combined_gem(gemspec, rbs_version_cache_key, combined_pins) + end + + # @param gemspec [Gem::Specification, Bundler::LazySpecification] + # @return [Array, nil] + def deserialize_combined_pin_cache gemspec + rbs_version_cache_key = lookup_rbs_version_cache_key(gemspec) + + combined = load_combined_gem(gemspec, rbs_version_cache_key) + return combined if combined + + # No combined cache entry exists yet for this gemspec - only + # `solargraph gems` writes one. See RbsMap#fallback_pins for which + # gems can supply a standalone substitute in the meantime. + # + # Deliberately not written to combined_pins_in_memory: that is + # process-wide and keyed only by name and version, so a + # provisional set stored there would go on being served after the + # build that supersedes it. + rbs_map = RbsMap.from_gemspec(gemspec, rbs_collection_path, rbs_collection_config_path) + fallback = rbs_map.fallback_pins + return nil unless fallback + + logger.debug { "Using #{gemspec.name}:#{gemspec.version}'s fallback RBS pins (no combined cache entry yet)" } + fallback + end + + # @param gemspec [Gem::Specification, Bundler::LazySpecification] + # @param out [StringIO, IO, nil] + # @return [void] + def uncache_gem gemspec, out: nil + PinCache.uncache(yardoc_path(gemspec), out: out) + PinCache.uncache(yard_gem_path(gemspec), out: out) + uncache_by_prefix(rbs_collection_pins_path_prefix(gemspec), out: out) + uncache_by_prefix(combined_path_prefix(gemspec), out: out) + rbs_version_cache_key = lookup_rbs_version_cache_key(gemspec) + combined_pins_in_memory.delete([gemspec.name, gemspec.version, rbs_version_cache_key]) + end + + # @param gemspec [Gem::Specification, Bundler::LazySpecification] + def yardoc_processing? gemspec + Yardoc.processing?(yardoc_path(gemspec)) + end + + # @return [Array] a list of possible standard library names + def possible_stdlibs + # all dirs and .rb files in Gem::RUBYGEMS_DIR + Dir.glob(File.join(Gem::RUBYGEMS_DIR, '*')).map do |file_or_dir| + basename = File.basename(file_or_dir) + # remove .rb + # @sg-ignore flow sensitive typing should be able to handle redefinition + basename = basename[0..-4] if basename.end_with?('.rb') + basename + end.sort.uniq + rescue StandardError => e + logger.info { "Failed to get possible stdlibs: #{e.message}" } + # @sg-ignore Need to add nil check here + logger.debug { e.backtrace.join("\n") } + [] + end + + private + + # @param gemspec [Gem::Specification, Bundler::LazySpecification] + # @param rebuild [Boolean] whether to rebuild the cache regardless of whether it already exists + # @param rbs_version_cache_key [String, nil] the cache key for the gem in the RBS collection + # + # @return [Array(Boolean, Boolean, Boolean)] whether to build YARD + # pins, RBS collection pins, and combined pins + def calculate_build_needs gemspec, rebuild:, rbs_version_cache_key: + if rebuild + build_yard = true + build_rbs_collection = true + build_combined = true + else + build_yard = !yard_gem?(gemspec) + build_rbs_collection = !rbs_collection_pins?(gemspec, rbs_version_cache_key) + # @sg-ignore Need to add nil check here + build_combined = !combined_gem?(gemspec, rbs_version_cache_key) || build_yard || build_rbs_collection + end + + build_yard = false if suppress_yard_cache?(gemspec, rbs_version_cache_key) + + [build_yard, build_rbs_collection, build_combined] + end + + # @param gemspec [Gem::Specification] + # @param rbs_version_cache_key [String, nil] + # @param build_yard [Boolean] + # @param build_rbs_collection [Boolean] + # @param build_combined [Boolean] + # @param out [StringIO, IO, nil] + # + # @return [void] + def build_combine_and_cache gemspec, + rbs_version_cache_key, + build_yard:, + build_rbs_collection:, + build_combined:, + out: + log_cache_info(gemspec, rbs_version_cache_key, + build_yard: build_yard, + build_rbs_collection: build_rbs_collection, + build_combined: build_combined, + out: out) + cache_yard_pins(gemspec, out) if build_yard + # this can be nil even if we aren't told to build it - see suppress_yard_cache? + yard_pins = deserialize_yard_pin_cache(gemspec) || [] + cache_rbs_collection_pins(gemspec, out) if build_rbs_collection + rbs_collection_pins = deserialize_rbs_collection_cache(gemspec, rbs_version_cache_key) || [] + cache_combined_pins(gemspec, rbs_version_cache_key, yard_pins, rbs_collection_pins) if build_combined + end + + # @param gemspec [Gem::Specification, Bundler::LazySpecification] + # @param rbs_version_cache_key [String, nil] + # @param build_yard [Boolean] + # @param build_rbs_collection [Boolean] + # @param build_combined [Boolean] + # @param out [StringIO, IO, nil] + # + # @return [void] + def log_cache_info gemspec, + rbs_version_cache_key, + build_yard:, + build_rbs_collection:, + build_combined:, + out: + type = [] + type << 'YARD' if build_yard + rbs_source_desc = RbsMap.rbs_source_desc(rbs_version_cache_key) + type << rbs_source_desc if build_rbs_collection && !rbs_source_desc.nil? + # we'll build it anyway, but it won't take long to build with + # only a single source + + # 'combining' is awkward terminology in this case + just_yard = build_yard && rbs_source_desc.nil? + + type << 'combined' if build_combined && !just_yard + out&.puts("Caching #{type.join(' and ')} pins for gem #{gemspec.name}:#{gemspec.version}") + end + + # @param gemspec [Gem::Specification] + # @param out [StringIO, IO, nil] + # + # @return [Array] + def cache_yard_pins gemspec, out + gem_yardoc_path = yardoc_path(gemspec) + Yardoc.build_docs(gem_yardoc_path, yard_plugins, gemspec) unless Yardoc.docs_built?(gem_yardoc_path) + pins = Yardoc.build_pins(gem_yardoc_path, gemspec, out: out) + serialize_yard_gem(gemspec, pins) + logger.info { "Cached #{pins.length} YARD pins for gem #{gemspec.name}:#{gemspec.version}" } unless pins.empty? + pins + end + + # @return [Hash{::Array => Array}] keyed by [gem name, gem version, RBS cache key] + def combined_pins_in_memory + PinCache.all_combined_pins_in_memory[yard_plugins] ||= {} + end + + # @param gemspec [Gem::Specification, Bundler::LazySpecification] + # @param _out [StringIO, IO, nil] + # @return [Array] + def cache_rbs_collection_pins gemspec, _out + rbs_map = RbsMap.from_gemspec(gemspec, rbs_collection_path, rbs_collection_config_path) + pins = rbs_map.pins + rbs_version_cache_key = rbs_map.cache_key + # cache pins even if result is zero, so we don't retry building pins + pins ||= [] + serialize_rbs_collection_pins(gemspec, rbs_version_cache_key, pins) + logger.info do + unless pins.empty? + "Cached #{pins.length} RBS collection pins for gem #{gemspec.name} #{gemspec.version} with " \ + "cache_key #{rbs_version_cache_key.inspect}" + end + end + pins + end + + # @param gemspec [Gem::Specification, Bundler::LazySpecification] + # @return [Array, nil] + def deserialize_yard_pin_cache gemspec + cached = load_yard_gem(gemspec) + if cached + cached + else + logger.debug "No YARD pin cache for #{gemspec.name}:#{gemspec.version}" + nil + end + end + + # @param gemspec [Gem::Specification, Bundler::LazySpecification] + # @param rbs_version_cache_key [String, nil] + # @return [Array, nil] + def deserialize_rbs_collection_cache gemspec, rbs_version_cache_key + cached = load_rbs_collection_pins(gemspec, rbs_version_cache_key) + Solargraph.assert_or_log(:pin_cache_rbs_collection, 'Asked for non-existent rbs collection') if cached.nil? + logger.info do + "Loaded #{cached&.length} pins from RBS collection cache for #{gemspec.name}:#{gemspec.version}" + end + cached + end + + # @return [Array] + def yard_path_components + ["yard-#{YARD::VERSION}", + yard_plugins.sort.uniq.join('-')] + end + + # @param gemspec [Gem::Specification, Bundler::LazySpecification] + # @return [String] + def yardoc_path gemspec + File.join(PinCache.base_dir, + *yard_path_components, + "#{gemspec.name}-#{gemspec.version}.yardoc") + end + + # @param gemspec [Gem::Specification, Bundler::LazySpecification] + # @return [String] + def yard_gem_path gemspec + File.join(PinCache.work_dir, *yard_path_components, "#{gemspec.name}-#{gemspec.version}.ser") + end + + # @param gemspec [Gem::Specification, Bundler::LazySpecification] + # @return [Array, nil] + def load_yard_gem gemspec + PinCache.load(yard_gem_path(gemspec)) + end + + # @param gemspec [Gem::Specification, Bundler::LazySpecification] + # @param pins [Array] + # @return [void] + def serialize_yard_gem gemspec, pins + PinCache.save(yard_gem_path(gemspec), pins) + end + + # @param gemspec [Gem::Specification, Bundler::LazySpecification] + # @return [Boolean] + def yard_gem? gemspec + exist?(yard_gem_path(gemspec)) + end + + # @param gemspec [Gem::Specification, Bundler::LazySpecification] + # @param hash [String, nil] + # @return [String] + def rbs_collection_pins_path gemspec, hash + rbs_collection_pins_path_prefix(gemspec) + "#{hash || 0}.ser" + end + + # @param gemspec [Gem::Specification, Bundler::LazySpecification] + # @return [String] + def rbs_collection_pins_path_prefix gemspec + File.join(PinCache.work_dir, 'rbs', "#{gemspec.name}-#{gemspec.version}-") + end + + # @param gemspec [Gem::Specification, Bundler::LazySpecification] + # @param hash [String, nil] + # + # @return [Array, nil] + def load_rbs_collection_pins gemspec, hash + PinCache.load(rbs_collection_pins_path(gemspec, hash)) + end + + # @param gemspec [Gem::Specification, Bundler::LazySpecification] + # @param hash [String, nil] + # @param pins [Array] + # @return [void] + def serialize_rbs_collection_pins gemspec, hash, pins + PinCache.save(rbs_collection_pins_path(gemspec, hash), pins) + end + + # @param gemspec [Gem::Specification, Bundler::LazySpecification] + # @param hash [String, nil] + # @return [String] + def combined_path gemspec, hash + File.join(combined_path_prefix(gemspec) + "-#{hash || 0}.ser") + end + + # @param gemspec [Gem::Specification, Bundler::LazySpecification] + # @return [String] + def combined_path_prefix gemspec + File.join(PinCache.work_dir, 'combined', yard_plugins.sort.join('-'), "#{gemspec.name}-#{gemspec.version}") + end + + # @param gemspec [Gem::Specification, Bundler::LazySpecification] + # @param hash [String, nil] + # @param pins [Array] + # @return [void] + def serialize_combined_gem gemspec, hash, pins + PinCache.save(combined_path(gemspec, hash), pins) + end + + # @param gemspec [Gem::Specification, Bundler::LazySpecification] + # @param hash [String] + def combined_gem? gemspec, hash + exist?(combined_path(gemspec, hash)) + end + + # @param gemspec [Gem::Specification, Bundler::LazySpecification] + # @param hash [String, nil] + # @return [Array, nil] + def load_combined_gem gemspec, hash + cached = combined_pins_in_memory[[gemspec.name, gemspec.version, hash]] + return cached if cached + loaded = PinCache.load(combined_path(gemspec, hash)) + combined_pins_in_memory[[gemspec.name, gemspec.version, hash]] = loaded if loaded + loaded + end + + # @param gemspec [Gem::Specification, Bundler::LazySpecification] + # @param hash [String, nil] + def rbs_collection_pins? gemspec, hash + exist?(rbs_collection_pins_path(gemspec, hash)) + end + + include Logging + + # @param path [String] + def exist? *path + File.file? File.join(*path) + end + + # @return [void] + # @param path_segments [Array] + # @param out [StringIO, IO, nil] + def uncache_by_prefix *path_segments, out: nil + path = File.join(*path_segments) + glob = "#{path}*" + out&.puts "Clearing pin cache in #{glob}" + Dir.glob(glob).each do |file| + next unless File.file?(file) + FileUtils.rm_rf file, secure: true + out&.puts "Clearing pin cache in #{file}" + end + end + class << self include Logging + # @return [Hash{Array => Hash{Array(String, String) => + # Array}}] yard plugins, then gemspec name and + # version + def all_combined_pins_in_memory + @all_combined_pins_in_memory ||= {} + end + # The base directory where cached YARD documentation and serialized pins are serialized # # @return [String] @@ -19,6 +460,47 @@ def base_dir File.join(Dir.home, '.cache', 'solargraph') end + # @param path_segments [Array] + # @param out [IO, nil] + # @return [void] + def uncache *path_segments, out: nil + path = File.join(*path_segments) + if File.exist?(path) + FileUtils.rm_rf path, secure: true + out&.puts "Clearing pin cache in #{path}" + else + out&.puts "Pin cache file #{path} does not exist" + end + end + + # @return [void] + # @param out [IO, nil] + # @param path_segments [Array] + def uncache_by_prefix *path_segments, out: nil + path = File.join(*path_segments) + glob = "#{path}*" + out&.puts "Clearing pin cache in #{glob}" + Dir.glob(glob).each do |file| + next unless File.file?(file) + FileUtils.rm_rf file, secure: true + out&.puts "Clearing pin cache in #{file}" + end + end + + # @param out [StringIO, IO, nil] + # @return [void] + def uncache_core out: nil + uncache(core_path, out: out) + # ApiMap keep this in memory + ApiMap.reset_core(out: out) + end + + # @param out [StringIO, IO, nil] + # @return [void] + def uncache_stdlib out: nil + uncache(stdlib_path, out: out) + end + # The working directory for the current Ruby, RBS, and Solargraph versions. # # @return [String] @@ -28,15 +510,6 @@ def work_dir File.join(base_dir, "ruby-#{RUBY_VERSION}", "rbs-#{RBS::VERSION}", "solargraph-#{Solargraph::VERSION}") end - # @param gemspec [Gem::Specification] - # @return [String] - def yardoc_path gemspec - File.join(base_dir, - "yard-#{YARD::VERSION}", - "yard-activesupport-concern-#{YARD::ActiveSupport::Concern::VERSION}", - "#{gemspec.name}-#{gemspec.version}.yardoc") - end - # @return [String] def stdlib_path File.join(work_dir, 'stdlib') @@ -96,12 +569,6 @@ def serialize_yard_gem gemspec, pins save(yard_gem_path(gemspec), pins) end - # @param gemspec [Gem::Specification] - # @return [Boolean] - def has_yard? gemspec - exist?(yard_gem_path(gemspec)) - end - # @param gemspec [Gem::Specification] # @param hash [String, nil] # @return [String] @@ -165,35 +632,13 @@ def has_rbs_collection? gemspec, hash exist?(rbs_collection_path(gemspec, hash)) end - # @return [void] - def uncache_core - uncache(core_path) - end - - # @return [void] - def uncache_stdlib - uncache(stdlib_path) - end - - # @param gemspec [Gem::Specification] - # @param out [IO, StringIO, nil] - # @return [void] - def uncache_gem gemspec, out: nil - uncache(yardoc_path(gemspec), out: out) - uncache_by_prefix(rbs_collection_path_prefix(gemspec), out: out) - uncache(yard_gem_path(gemspec), out: out) - uncache_by_prefix(combined_path_prefix(gemspec), out: out) - end - # @return [void] def clear FileUtils.rm_rf base_dir, secure: true end - private - # @param file [String] - # @sg-ignore Marshal.load returns Object; we know it's Array + # @sg-ignore Marshal.load evaluates to boolean here which is wrong # @return [Array, nil] def load file return nil unless File.file?(file) @@ -204,11 +649,6 @@ def load file nil end - # @param path [String] - def exist? *path - File.file? File.join(*path) - end - # @param file [String] # @param pins [Array] # @return [void] @@ -220,28 +660,19 @@ def save file, pins logger.debug { "Cache#save: Saved #{pins.length} pins to #{file}" } end - # @param path_segments [Array] - # @return [void] - # @param [Object, nil] out - def uncache *path_segments, out: nil - path = File.join(*path_segments) - return unless File.exist?(path) - FileUtils.rm_rf path, secure: true - out&.puts "Clearing pin cache in #{path}" + def core? + File.file?(core_path) end - # @return [void] - # @param path_segments [Array] - # @param [Object, nil] out - def uncache_by_prefix *path_segments, out: nil - path = File.join(*path_segments) - glob = "#{path}*" - out&.puts "Clearing pin cache in #{glob}" - Dir.glob(glob).each do |file| - next unless File.file?(file) - FileUtils.rm_rf file, secure: true - out&.puts "Clearing pin cache in #{file}" - end + # @param out [StringIO, IO, nil] + # @return [Array] + def cache_core out: $stderr + RbsMap::CoreMap.new.cache_core(out: out) + end + + # @param path [String] + def exist? *path + File.file? File.join(*path) end end end diff --git a/lib/solargraph/rbs_map.rb b/lib/solargraph/rbs_map.rb index c86dc6b74..4d25c821f 100644 --- a/lib/solargraph/rbs_map.rb +++ b/lib/solargraph/rbs_map.rb @@ -116,9 +116,13 @@ def self.from_gemspec gemspec, rbs_collection_path, rbs_collection_config_path return rbs_map if rbs_map.resolved? # try any version of the gem in the collection - RbsMap.new(gemspec.name, nil, - rbs_collection_paths: [rbs_collection_path].compact, - rbs_collection_config_path: rbs_collection_config_path) + rbs_map = RbsMap.new(gemspec.name, nil, + rbs_collection_paths: [rbs_collection_path].compact, + rbs_collection_config_path: rbs_collection_config_path) + + return rbs_map if rbs_map.resolved? + + StdlibMap.new(gemspec.name) end # @param out [IO, nil] where to log messages @@ -153,6 +157,19 @@ def resolved? @resolved end + # A standalone substitute for this gem's PinCache-combined pins + # (PinCache#deserialize_combined_pin_cache), for a caller that needs + # something before that combined cache has been built. Base RbsMap + # has none to offer, since a combined cache is a merge of this map's + # pins with separately-cached YARD pins and dropping the YARD half + # isn't safe in general. RbsMap::StdlibMap overrides this because its + # own pins need no such merge. + # + # @return [Array, nil] + def fallback_pins + nil + end + # @return [RBS::Repository] def repository @repository ||= RBS::Repository.new(no_stdlib: false).tap do |repo| diff --git a/lib/solargraph/rbs_map/conversions.rb b/lib/solargraph/rbs_map/conversions.rb index ebe7a6ce0..290a47315 100644 --- a/lib/solargraph/rbs_map/conversions.rb +++ b/lib/solargraph/rbs_map/conversions.rb @@ -164,9 +164,9 @@ def fqns type_name # @return [void] def convert_self_type_to_pins decl, closure type = build_type(decl.name, decl.args) - generic_values = type.all_params.map(&:to_s) + generic_values = type.all_params.map(&:rooted_tags) include_pin = Solargraph::Pin::Reference::Include.new( - name: decl.name.relative!.to_s, + name: type.name, type_location: location_decl_to_pin_location(decl.location), generic_values: generic_values, closure: closure, @@ -279,8 +279,7 @@ def class_decl_to_pin decl pins.push class_pin if decl.super_class type = build_type(decl.super_class.name, decl.super_class.args) - generic_values = type.all_params.map(&:to_s) - superclass_name = decl.super_class.name.to_s + generic_values = type.all_params.map(&:rooted_tags) pins.push Solargraph::Pin::Reference::Superclass.new( type_location: location_decl_to_pin_location(decl.super_class.location), closure: class_pin, @@ -299,7 +298,7 @@ def interface_decl_to_pin decl class_pin = Solargraph::Pin::Namespace.new( type: :module, type_location: location_decl_to_pin_location(decl.location), - name: decl.name.relative!.to_s, + name: fqns(decl.name), closure: Solargraph::Pin::ROOT_PIN, comments: decl.comment&.string, generics: type_parameter_names(decl), @@ -318,7 +317,7 @@ def interface_decl_to_pin decl def module_decl_to_pin decl module_pin = Solargraph::Pin::Namespace.new( type: :module, - name: decl.name.relative!.to_s, + name: fqns(decl.name), type_location: location_decl_to_pin_location(decl.location), closure: Solargraph::Pin::ROOT_PIN, comments: decl.comment&.string, @@ -514,24 +513,23 @@ def method_def_to_pin decl, closure, context pin.instance_variable_set(:@return_type, ComplexType::VOID) end end - if decl.singleton? - final_scope = :class - name = decl.name.to_s - visibility = calculate_method_visibility(decl, context, closure, final_scope, name) - pin = Solargraph::Pin::Method.new( - name: name, - closure: closure, - comments: decl.comment&.string, - type_location: location_decl_to_pin_location(decl.location), - visibility: visibility, - scope: final_scope, - signatures: [], - generics: generics, - source: :rbs - ) - pin.signatures.concat method_def_to_sigs(decl, pin) - pins.push pin - end + return unless decl.singleton? + final_scope = :class + name = decl.name.to_s + visibility = calculate_method_visibility(decl, context, closure, final_scope, name) + pin = Solargraph::Pin::Method.new( + name: name, + closure: closure, + comments: decl.comment&.string, + type_location: location_decl_to_pin_location(decl.location), + visibility: visibility, + scope: final_scope, + signatures: [], + generics: generics, + source: :rbs + ) + pin.signatures.concat method_def_to_sigs(decl, pin) + pins.push pin end # @param decl [RBS::AST::Members::MethodDefinition] @@ -551,19 +549,23 @@ def method_def_to_sigs decl, pin Pin::Signature.new(generics: generics, parameters: block_parameters, return_type: block_return_type, source: :rbs, type_location: type_location, closure: pin) end - Pin::Signature.new(generics: generics, parameters: signature_parameters, return_type: signature_return_type, block: block, source: :rbs, + Pin::Signature.new(generics: generics, parameters: signature_parameters, + return_type: signature_return_type, block: block, source: :rbs, type_location: type_location, closure: pin) end end # @param location [RBS::Location, nil] # @return [Solargraph::Location, nil] - def location_decl_to_pin_location(location) + def location_decl_to_pin_location location return nil if location&.name.nil? + # @sg-ignore flow sensitive typing should handle return nil if location&.name.nil? start_pos = Position.new(location.start_line - 1, location.start_column) + # @sg-ignore flow sensitive typing should handle return nil if location&.name.nil? end_pos = Position.new(location.end_line - 1, location.end_column) range = Range.new(start_pos, end_pos) + # @sg-ignore flow sensitve typing should handle return nil if location&.name.nil? Location.new(location.name.to_s, range) end @@ -574,7 +576,7 @@ def location_decl_to_pin_location(location) def parts_of_function type, pin, implicit_nil [ RbsTranslator.to_parameter_pins(type, pin, pin.parameter_names), - extract_method_type_return_type(type, implicit_nil).force_rooted + extract_method_type_return_type(type, implicit_nil: implicit_nil).force_rooted ] end @@ -701,9 +703,9 @@ def civar_to_pin decl, closure # @return [void] def include_to_pin decl, closure type = build_type(decl.name, decl.args) - generic_values = type.all_params.map(&:to_s) + generic_values = type.all_params.map(&:rooted_tags) pins.push Solargraph::Pin::Reference::Include.new( - name: decl.name.relative!.to_s, + name: type.rooted_name, # reference pins use rooted names type_location: location_decl_to_pin_location(decl.location), generic_values: generic_values, closure: closure, @@ -718,8 +720,9 @@ def prepend_to_pin decl, closure type = build_type(decl.name, decl.args) generic_values = type.all_params.map(&:rooted_tags) pins.push Solargraph::Pin::Reference::Prepend.new( - name: decl.name.relative!.to_s, + name: type.rooted_name, # reference pins use rooted names type_location: location_decl_to_pin_location(decl.location), + generic_values: generic_values, closure: closure, source: :rbs ) @@ -732,8 +735,9 @@ def extend_to_pin decl, closure type = build_type(decl.name, decl.args) generic_values = type.all_params.map(&:rooted_tags) pins.push Solargraph::Pin::Reference::Extend.new( - name: decl.name.relative!.to_s, + name: type.rooted_name, # reference pins use rooted names type_location: location_decl_to_pin_location(decl.location), + generic_values: generic_values, closure: closure, source: :rbs ) @@ -760,7 +764,7 @@ def alias_to_pin decl, closure 'int' => 'Integer', 'untyped' => '', 'NilClass' => 'nil' - } + }.freeze private_constant :RBS_TO_YARD_TYPE # Extract a ComplexType from a MethodType's return type. @@ -768,17 +772,18 @@ def alias_to_pin decl, closure # This method will convert type aliases to concrete types. # # @param type [RBS::MethodType] + # @param implicit_nil [Boolean] # @return [ComplexType] - def extract_method_type_return_type type, implicit_nil - tag = RbsTranslator.to_complex_type(type.type.return_type) - return ComplexType.parse("#{tag}, nil") if tag && implicit_nil - tag + def extract_method_type_return_type type, implicit_nil: + tag = RbsTranslator.to_complex_type(type.type.return_type) + return ComplexType.parse("#{tag}, nil") if tag && implicit_nil + tag end # @param type_name [RBS::TypeName] # @param type_args [Enumerable] # @return [ComplexType::UniqueType] - def build_type(type_name, type_args = []) + def build_type type_name, type_args = [] base = RBS_TO_YARD_TYPE[type_name.relative!.to_s] || type_name.relative!.to_s params = type_args.map { |arg| RbsTranslator.to_complex_type(arg).force_rooted } if base == 'Hash' && params.length == 2 @@ -797,9 +802,9 @@ def add_mixins decl, namespace # @todo are we handling prepend correctly? klass = mixin.is_a?(RBS::AST::Members::Include) ? Pin::Reference::Include : Pin::Reference::Extend type = build_type(mixin.name, mixin.args) - generic_values = type.all_params.map(&:to_s) + generic_values = type.all_params.map(&:rooted_tags) pins.push klass.new( - name: mixin.name.relative!.to_s, + name: type.rooted_name, # reference pins use rooted names type_location: location_decl_to_pin_location(mixin.location), generic_values: generic_values, closure: namespace, diff --git a/lib/solargraph/rbs_map/stdlib_map.rb b/lib/solargraph/rbs_map/stdlib_map.rb index e6ebcf90f..559a50075 100644 --- a/lib/solargraph/rbs_map/stdlib_map.rb +++ b/lib/solargraph/rbs_map/stdlib_map.rb @@ -61,6 +61,11 @@ def resolve_dependencies? true end + # @return [Array, nil] + def fallback_pins + pins if resolved? + end + # @param library [String] # @return [StdlibMap] def self.load library diff --git a/lib/solargraph/shell.rb b/lib/solargraph/shell.rb index 89859da21..38c668c97 100755 --- a/lib/solargraph/shell.rb +++ b/lib/solargraph/shell.rb @@ -109,20 +109,8 @@ def clear # @param gem [String] # @param version [String, nil] def cache gem, version = nil - gemspec = Gem::Specification.find_by_name(gem, version) - - if options[:rebuild] || !PinCache.has_yard?(gemspec) - pins = GemPins.build_yard_pins(['yard-activesupport-concern'], gemspec) - PinCache.serialize_yard_gem(gemspec, pins) - end - - workspace = Solargraph::Workspace.new(Dir.pwd) if File.exist?('rbs_collection.yaml') - rbs_map = RbsMap.from_gemspec(gemspec, workspace&.rbs_collection_path, workspace&.rbs_collection_config_path) - if options[:rebuild] || !PinCache.has_rbs_collection?(gemspec, rbs_map.cache_key) - PinCache.serialize_rbs_collection_gem(gemspec, rbs_map.cache_key, rbs_map.pins) - end - rescue Gem::MissingSpecError - warn "Gem '#{gem}' not found" + gems(gem + (version ? "=#{version}" : '')) + # ' end desc 'uncache GEM [...GEM]', 'Delete specific cached gem documentation' @@ -135,19 +123,24 @@ def cache gem, version = nil # @return [void] def uncache *gems raise ArgumentError, 'No gems specified.' if gems.empty? + workspace = Solargraph::Workspace.new(Dir.pwd) + gems.each do |gem| if gem == 'core' - PinCache.uncache_core + PinCache.uncache_core(out: $stdout) next end if gem == 'stdlib' - PinCache.uncache_stdlib + PinCache.uncache_stdlib(out: $stdout) next end - spec = Gem::Specification.find_by_name(gem) - PinCache.uncache_gem(spec, out: $stdout) + spec = workspace.find_gem(gem) + raise Thor::InvocationError, "Gem '#{gem}' not found" if spec.nil? + + # @sg-ignore flow sensitive typing needs to handle 'raise if' + workspace.uncache_gem(spec, out: $stdout) end end @@ -183,14 +176,12 @@ def gems *names workspace = Solargraph::Workspace.new('.') if names.empty? - Gem::Specification.to_a.each { |spec| do_cache spec, rebuild: options[:rebuild] } - $stderr.puts "Documentation cached for all #{Gem::Specification.count} gems." + workspace.cache_all_for_workspace!($stdout, rebuild: options[:rebuild]) else warn("Caching these gems: #{names}") names.each do |name| if name == 'core' - # @sg-ignore cache_core and core? are dynamically defined - PinCache.cache_core(out: $stdout) # if !PinCache.core? || options[:rebuild] + PinCache.cache_core(out: $stdout) if !PinCache.core? || options[:rebuild] next end @@ -198,18 +189,7 @@ def gems *names if gemspec.nil? warn "Gem '#{name}' not found" else - if options[:rebuild] || !PinCache.has_yard?(gemspec) - pins = GemPins.build_yard_pins(['yard-activesupport-concern'], gemspec) - PinCache.serialize_yard_gem(gemspec, pins) - end - - workspace = Solargraph::Workspace.new(Dir.pwd) - rbs_map = RbsMap.from_gemspec(gemspec, workspace.rbs_collection_path, workspace.rbs_collection_config_path) - if options[:rebuild] || !PinCache.has_rbs_collection?(gemspec, rbs_map.cache_key) - # cache pins even if result is zero, so we don't retry building pins - pins = rbs_map.pins || [] - PinCache.serialize_rbs_collection_gem(gemspec, rbs_map.cache_key, pins) - end + workspace.cache_gem(gemspec, rebuild: options[:rebuild], out: $stdout) end rescue Gem::MissingSpecError warn "Gem '#{name}' not found" @@ -596,27 +576,5 @@ def print_pin pin puts pin.inspect end end - - # @param gemspec [Gem::Specification, nil] - # @param rebuild [Boolean] - # @return [void] - def do_cache gemspec, rebuild: false - if gemspec.nil? - warn "Gem '#{gemspec&.name}' not found" - else - if rebuild || !PinCache.has_yard?(gemspec) - pins = GemPins.build_yard_pins(['yard-activesupport-concern'], gemspec) - PinCache.serialize_yard_gem(gemspec, pins) - end - - workspace = Solargraph::Workspace.new(Dir.pwd) - rbs_map = RbsMap.from_gemspec(gemspec, workspace.rbs_collection_path, workspace.rbs_collection_config_path) - if rebuild || !PinCache.has_rbs_collection?(gemspec, rbs_map.cache_key) - # cache pins even if result is zero, so we don't retry building pins - pins = rbs_map.pins || [] - PinCache.serialize_rbs_collection_gem(gemspec, rbs_map.cache_key, pins) - end - end - end end end diff --git a/lib/solargraph/workspace.rb b/lib/solargraph/workspace.rb index d3346c9b4..ce338bbfb 100644 --- a/lib/solargraph/workspace.rb +++ b/lib/solargraph/workspace.rb @@ -2,6 +2,7 @@ require 'open3' require 'json' +require 'yaml' module Solargraph # A workspace consists of the files in a project's directory and the @@ -9,6 +10,8 @@ module Solargraph # in an associated Library or ApiMap. # class Workspace + include Logging + autoload :Config, 'solargraph/workspace/config' autoload :Gemspecs, 'solargraph/workspace/gemspecs' autoload :RequirePaths, 'solargraph/workspace/require_paths' @@ -16,11 +19,8 @@ class Workspace # @return [String] attr_reader :directory - # @return [Array] - attr_reader :gemnames - alias source_gems gemnames - - # @param directory [String] TODO: Remove '' and '*' special cases + # @todo Remove '' and '*' special cases + # @param directory [String] # @param config [Config, nil] # @param server [Hash] def initialize directory = '', config = nil, server = {} @@ -34,7 +34,6 @@ def initialize directory = '', config = nil, server = {} @config = config @server = server load_sources - @gemnames = [] require_plugins end @@ -51,6 +50,69 @@ def config @config ||= Solargraph::Workspace::Config.new(directory) end + # @param stdlib_name [String] + # + # @return [Array] + def stdlib_dependencies stdlib_name + gemspecs.stdlib_dependencies(stdlib_name) + end + + # @param out [IO, nil] output stream for logging + # @param gemspec [Gem::Specification] + # @return [Array] + def fetch_dependencies gemspec, out: $stderr + gemspecs.fetch_dependencies(gemspec, out: out) + end + + # @param require [String] The string sent to 'require' in the code to resolve, e.g. 'rails', 'bundler/require' + # + # @return [Array, nil] + def resolve_require require + gemspecs.resolve_require(require) + end + + # @return [Solargraph::PinCache] + def pin_cache + @pin_cache ||= fresh_pincache + end + + # @return [Environ] + def global_environ + # empty docmap, since the result needs to work in any possible + # context here + @global_environ ||= Convention.for_global(DocMap.new([], self, out: nil)) + end + + # @param gemspec [Gem::Specification] + # @param out [StringIO, IO, nil] output stream for logging + # @param rebuild [Boolean] whether to rebuild the pins even if they are cached + # + # @return [void] + def cache_gem gemspec, out: nil, rebuild: false + pin_cache.cache_gem(gemspec: gemspec, out: out, rebuild: rebuild) + end + + # @param gemspec [Gem::Specification, Bundler::LazySpecification] + # @param out [StringIO, IO, nil] output stream for logging + # + # @return [void] + def uncache_gem gemspec, out: nil + pin_cache.uncache_gem(gemspec, out: out) + end + + # @return [Solargraph::PinCache] + def fresh_pincache + PinCache.new(rbs_collection_path: rbs_collection_path, + rbs_collection_config_path: rbs_collection_config_path, + yard_plugins: yard_plugins, + directory: directory) + end + + # @return [Array] + def yard_plugins + @yard_plugins ||= global_environ.yard_plugins.sort.uniq + end + # @param level [Symbol] # @return [TypeChecker::Rules] def rules level @@ -64,6 +126,7 @@ def rules level # @param sources [Array] # @return [Boolean] True if the source was added to the workspace def merge *sources + # @sg-ignore Need to add nil check here unless directory == '*' || sources.all? { |source| source_hash.key?(source.filename) } # Reload the config to determine if a new source should be included @config = Solargraph::Workspace::Config.new(directory) @@ -128,6 +191,31 @@ def would_require? path false end + # True if the workspace has a root Gemfile. + # + # @todo Handle projects with custom Bundler/Gemfile setups (see DocMap#gemspecs_required_from_bundler) + # + def gemfile? + directory && File.file?(File.join(directory, 'Gemfile')) + end + + # True if the workspace contains at least one gemspec file. + # + # @return [Boolean] + def gemspec? + !gemspec_files.empty? + end + + # Get an array of all gemspec files in the workspace. + # + # @return [Array] + def gemspec_files + return [] if directory.empty? || directory == '*' + @gemspec_files ||= Dir[File.join(directory, '**/*.gemspec')].select do |gs| + config.allow? gs + end + end + # @return [String, nil] def rbs_collection_path @rbs_collection_path ||= read_rbs_collection_path @@ -147,7 +235,34 @@ def rbs_collection_config_path # # @return [Gem::Specification, nil] def find_gem name, version = nil, out: nil - Gem::Specification.find_by_name(name, version) + gemspecs.find_gem(name, version, out: out) + end + + # @return [Array] + def all_gemspecs_from_bundle + gemspecs.all_gemspecs_from_bundle + end + + # @param out [StringIO, IO, nil] output stream for logging + # @param rebuild [Boolean] whether to rebuild the pins even if they are cached + # @return [void] + def cache_all_for_workspace! out, rebuild: false + PinCache.cache_core(out: out) unless PinCache.core? && !rebuild + + gem_specs = all_gemspecs_from_bundle + # try any possible standard libraries, but be quiet about it + stdlib_specs = pin_cache.possible_stdlibs.map { |stdlib| find_gem(stdlib, out: nil) }.compact + specs = (gem_specs + stdlib_specs) + specs.each do |spec| + pin_cache.cache_gem(gemspec: spec, rebuild: rebuild, out: out) unless pin_cache.cached?(spec) + end + out&.puts "Documentation cached for all #{specs.length} gems." + + # do this after so that we prefer stdlib requires from gems, + # which are likely to be newer and have more pins + pin_cache.cache_all_stdlibs(out: out, rebuild: rebuild) + + out&.puts 'Documentation cached for core, standard library and gems.' end # Synchronize the workspace from the provided updater. @@ -160,6 +275,7 @@ def synchronize! updater # @sg-ignore return type could not be inferred # @return [String] + # @sg-ignore Need to validate config def command_path server['commandPath'] || 'solargraph' end @@ -170,29 +286,9 @@ def directory_or_nil directory end - # True if the workspace has a root Gemfile. - # - # @todo Handle projects with custom Bundler/Gemfile setups (see DocMap#gemspecs_required_from_bundler) - # - def gemfile? - directory && File.file?(File.join(directory, 'Gemfile')) - end - - # True if the workspace contains at least one gemspec file. - # - # @return [Boolean] - def gemspec? - !gemspec_files.empty? - end - - # Get an array of all gemspec files in the workspace. - # - # @return [Array] - def gemspec_files - return [] if directory.empty? || directory == '*' - @gemspec_files ||= Dir[File.join(directory, '**/*.gemspec')].select do |gs| - config.allow? gs - end + # @return [Solargraph::Workspace::Gemspecs] + def gemspecs + @gemspecs ||= Solargraph::Workspace::Gemspecs.new(directory_or_nil) end private diff --git a/lib/solargraph/workspace/gemspecs.rb b/lib/solargraph/workspace/gemspecs.rb index 2c29b948c..1312dc00d 100644 --- a/lib/solargraph/workspace/gemspecs.rb +++ b/lib/solargraph/workspace/gemspecs.rb @@ -44,8 +44,7 @@ def resolve_require require return auto_required_gemspecs_from_bundler if require == 'bundler/require' # Determine gem name based on the require path - file = "lib/#{require}.rb" - spec_with_path = Gem::Specification.find_by_path(file) + spec_with_path = Gem::Specification.find_by_path(require) all_gemspecs = all_gemspecs_from_bundle @@ -73,6 +72,7 @@ def resolve_require require # look ourselves just in case this is hanging out somewhere # that find_by_path doesn't index + file = "lib/#{require}.rb" gemspec = all_gemspecs.find do |spec| spec = to_gem_specification(spec) unless spec.respond_to?(:files) diff --git a/lib/solargraph/yardoc.rb b/lib/solargraph/yardoc.rb index 2150dcbef..4accf9425 100644 --- a/lib/solargraph/yardoc.rb +++ b/lib/solargraph/yardoc.rb @@ -8,15 +8,15 @@ module Solargraph module Yardoc module_function - # Build and cache a gem's yardoc and return the path. If the cache already - # exists, do nothing and return the path. + # Build and save a gem's yardoc into a given path. # - # @param yard_plugins [Array] The names of YARD plugins to use. + # @param gem_yardoc_path [String] the path to the yardoc cache of a particular gem + # @param yard_plugins [Array] # @param gemspec [Gem::Specification] - # @return [String] The path to the cached yardoc. - def cache yard_plugins, gemspec - path = PinCache.yardoc_path gemspec - return path if cached?(gemspec) + # + # @return [void] + def build_docs gem_yardoc_path, yard_plugins, gemspec + return if docs_built?(gem_yardoc_path) unless Dir.exist? gemspec.gem_dir # Can happen in at least some (old?) RubyGems versions when we @@ -24,37 +24,44 @@ def cache yard_plugins, gemspec # # https://github.com/apiology/solargraph/actions/runs/17650140201/job/50158676842?pr=10 Solargraph.logger.info { "Bad info from gemspec - #{gemspec.gem_dir} does not exist" } - return path + return end Solargraph.logger.info "Caching yardoc for #{gemspec.name} #{gemspec.version}" - cmd = "yardoc --db #{path} --no-output --plugin solargraph" + cmd = "yardoc --db #{gem_yardoc_path} --no-output --plugin solargraph" yard_plugins.each { |plugin| cmd << " --plugin #{plugin}" } Solargraph.logger.debug { "Running: #{cmd}" } # @todo set these up to run in parallel - # @todo Is the chdir argument being used here? - # @sg-ignore Unrecognized keyword argument chdir to Open3.capture2e + # @sg-ignore Our fill won't work properly due to an issue in + # Callable#arity_matches? - see comment there stdout_and_stderr_str, status = Open3.capture2e(current_bundle_env_tweaks, cmd, chdir: gemspec.gem_dir) - unless status.success? - Solargraph.logger.warn { "YARD failed running #{cmd.inspect} in #{gemspec.gem_dir}" } - Solargraph.logger.info stdout_and_stderr_str - end - path + return if status.success? + Solargraph.logger.warn { "YARD failed running #{cmd.inspect} in #{gemspec.gem_dir}" } + Solargraph.logger.info stdout_and_stderr_str + end + + # @param gem_yardoc_path [String] the path to the yardoc cache of a particular gem + # @param gemspec [Gem::Specification, Bundler::LazySpecification] + # @param out [StringIO, IO, nil] where to log messages + # @return [Array] + def build_pins gem_yardoc_path, gemspec, out: $stderr + yardoc = load!(gem_yardoc_path) + YardMap::Mapper.new(yardoc, gemspec).map end # True if the gem yardoc is cached. # - # @param gemspec [Gem::Specification] - def cached? gemspec - yardoc = File.join(PinCache.yardoc_path(gemspec), 'complete') + # @param gem_yardoc_path [String] + def docs_built? gem_yardoc_path + yardoc = File.join(gem_yardoc_path, 'complete') File.exist?(yardoc) end # True if another process is currently building the yardoc cache. # - # @param gemspec [Gem::Specification] - def processing? gemspec - yardoc = File.join(PinCache.yardoc_path(gemspec), 'processing') + # @param gem_yardoc_path [String] the path to the yardoc cache of a particular gem + def processing? gem_yardoc_path + yardoc = File.join(gem_yardoc_path, 'processing') File.exist?(yardoc) end @@ -62,10 +69,10 @@ def processing? gemspec # # @note This method modifies the global YARD registry. # - # @param gemspec [Gem::Specification] + # @param gem_yardoc_path [String] the path to the yardoc cache of a particular gem # @return [Array] - def load! gemspec - YARD::Registry.load! PinCache.yardoc_path gemspec + def load! gem_yardoc_path + YARD::Registry.load! gem_yardoc_path YARD::Registry.all end @@ -80,7 +87,7 @@ def load! gemspec # @return [Hash{String => String}] a hash of environment variables to override def current_bundle_env_tweaks tweaks = {} - # @sg-ignore Unresolved call to empty? on String, nil + # @sg-ignore Translate to something flow sensitive typing understands if ENV['BUNDLE_GEMFILE'] && !ENV['BUNDLE_GEMFILE'].empty? tweaks['BUNDLE_GEMFILE'] = File.expand_path(ENV['BUNDLE_GEMFILE']) end diff --git a/spec/api_map_method_spec.rb b/spec/api_map_method_spec.rb index 063b22f32..e82325256 100644 --- a/spec/api_map_method_spec.rb +++ b/spec/api_map_method_spec.rb @@ -143,10 +143,10 @@ class B describe '#cache_all_for_doc_map!' do it 'can cache gems without a bench' do api_map = described_class.new - doc_map = instance_double(Solargraph::DocMap, cache_all!: true) + doc_map = instance_double(Solargraph::DocMap, cache_doc_map_gems!: true) allow(Solargraph::DocMap).to receive(:new).and_return(doc_map) api_map.cache_all_for_doc_map!(out: $stderr) - expect(doc_map).to have_received(:cache_all!).with($stderr, rebuild: false) + expect(doc_map).to have_received(:cache_doc_map_gems!).with($stderr, rebuild: false) end end diff --git a/spec/doc_map_spec.rb b/spec/doc_map_spec.rb index 2dbe28fb7..8fdf815a2 100644 --- a/spec/doc_map_spec.rb +++ b/spec/doc_map_spec.rb @@ -19,7 +19,7 @@ let(:plain_doc_map) { described_class.new([], workspace, out: nil) } before do - doc_map.cache_all!(nil) if pre_cache + doc_map.cache_doc_map_gems!(nil) if pre_cache end context 'with a require in solargraph test bundle' do @@ -67,18 +67,48 @@ end end - it 'does not warn for redundant requires' do - # Requiring 'set' is unnecessary because it's already included in core. It - # might make sense to log redundant requires, but a warning is overkill. - allow(Solargraph.logger).to receive(:warn).and_call_original - described_class.new(['set'], workspace) - expect(Solargraph.logger).not_to have_received(:warn).with(/path set/) + context 'when deserialization takes a while' do + let(:pre_cache) { false } + let(:requires) { ['backport'] } + + before do + # proxy this method to simulate a long-running deserialization + allow(Benchmark).to receive(:measure) do |&block| + block.call + 5.0 + end + end + + it 'logs timing' do + # force lazy evaluation + _pins = doc_map.pins + expect(out.string).to include('Deserialized ').and include(' gem pins ').and include(' ms') + end + end + + context 'with an uncached but valid gemspec' do + let(:requires) { ['uncached_gem'] } + let(:pre_cache) { false } + let(:workspace) { instance_double(Solargraph::Workspace) } + + it 'tracks uncached_gemspecs' do + pincache = instance_double(Solargraph::PinCache, cache_stdlib_rbs_map: false) + uncached_gemspec = Gem::Specification.new('uncached_gem', '1.0.0') + allow(workspace).to receive(:fetch_dependencies).with(uncached_gemspec, out: out).and_return([]) + allow(workspace).to receive_messages(fresh_pincache: pincache, resolve_require: [uncached_gemspec], + stdlib_dependencies: [], global_environ: Solargraph::Environ.new) + allow(Gem::Specification).to receive(:find_by_path).with('uncached_gem').and_return(uncached_gemspec) + allow(workspace).to receive(:global_environ).and_return(Solargraph::Environ.new) + allow(pincache).to receive(:deserialize_combined_pin_cache).with(uncached_gemspec).and_return(nil) + + expect(doc_map.uncached_gemspecs).to eq([uncached_gemspec]) + end end context 'with require as bundle/require' do it 'imports all gems when bundler/require used' do doc_map_with_bundler_require = described_class.new(['bundler/require'], workspace, out: nil) - doc_map_with_bundler_require.cache_all!(nil) + doc_map_with_bundler_require.cache_doc_map_gems!(nil) expect(doc_map_with_bundler_require.pins.length - plain_doc_map.pins.length).to be_positive end end diff --git a/spec/gem_pins_spec.rb b/spec/gem_pins_spec.rb index 9d8101d17..944afd331 100644 --- a/spec/gem_pins_spec.rb +++ b/spec/gem_pins_spec.rb @@ -6,7 +6,7 @@ let(:pin) { doc_map.pins.find { |pin| pin.path == path } } before do - doc_map.cache_all!(STDERR) # rubocop:disable Style/GlobalStdStream + doc_map.cache_doc_map_gems!(STDERR) # rubocop:disable Style/GlobalStdStream end context 'with a combined method pin' do diff --git a/spec/pin_cache_spec.rb b/spec/pin_cache_spec.rb new file mode 100644 index 000000000..1396a80ca --- /dev/null +++ b/spec/pin_cache_spec.rb @@ -0,0 +1,244 @@ +# frozen_string_literal: true + +require 'bundler' +require 'benchmark' +require 'tmpdir' + +describe Solargraph::PinCache do + subject(:pin_cache) do + described_class.new(rbs_collection_path: '.gem_rbs_collection', + rbs_collection_config_path: 'rbs_collection.yaml', + directory: Dir.pwd, + yard_plugins: ['activesupport-concern']) + end + + describe '#cached?' do + it 'returns true for a gem that is cached' do + allow(File).to receive(:file?).with(%r{.*stdlib/backport.ser$}).and_return(false) + allow(File).to receive(:file?).with(%r{.*combined/.*/backport-.*.ser$}).and_return(true) + + gemspec = Gem::Specification.find_by_name('backport') + expect(pin_cache.cached?(gemspec)).to be true + end + + it 'returns false for a gem that is not cached' do + gemspec = Gem::Specification.new.tap do |spec| + spec.name = 'nonexistent' + spec.version = '0.0.1' + end + expect(pin_cache.cached?(gemspec)).to be false + end + end + + describe '.core?' do + it 'returns true when core pins exist' do + allow(File).to receive(:file?).with(%r{.*/core.ser$}).and_return(true) + + expect(described_class.core?).to be true + end + + it "returns true when core pins don't" do + allow(File).to receive(:file?).with(%r{.*/core.ser$}).and_return(false) + + expect(described_class.core?).to be false + end + end + + describe '#possible_stdlibs' do + it 'is tolerant of less usual Ruby installations' do + stub_const('Gem::RUBYGEMS_DIR', nil) + + expect(pin_cache.possible_stdlibs).to eq([]) + end + end + + describe '#cache_all_stdlibs' do + it 'creates stdlibmaps' do + allow(Solargraph::RbsMap::StdlibMap).to receive(:new).and_return(instance_double(Solargraph::RbsMap::StdlibMap)) + + pin_cache.cache_all_stdlibs + + expect(Solargraph::RbsMap::StdlibMap).to have_received(:new).at_least(:once) + end + end + + describe '#cache_gem' do + context 'with an already in-memory gem' do + let(:backport_gemspec) { Gem::Specification.find_by_name('backport') } + + before do + pin_cache.cache_gem(gemspec: backport_gemspec, out: nil) + end + + it 'does not load the gem again' do + allow(Marshal).to receive(:load).and_call_original + + pin_cache.cache_gem(gemspec: backport_gemspec, out: nil) + + expect(Marshal).not_to have_received(:load).with(anything) + end + end + + context 'with the parser gem' do + before do + pin_cache.uncache_gem(Gem::Specification.find_by_name('parser'), out: nil) + allow(Solargraph::Yardoc).to receive(:build_docs) + end + + it 'chooses not to use YARD' do + parser_gemspec = Gem::Specification.find_by_name('parser') + pin_cache.cache_gem(gemspec: parser_gemspec, out: nil) + # if this fails, you may not have run `bundle exec rbs collection update` + expect(Solargraph::Yardoc).not_to have_received(:build_docs).with(any_args) + end + end + + context 'with an installed gem' do + before do + pin_cache.cache_gem(gemspec: Gem::Specification.find_by_name('kramdown'), out: nil) + end + + it 'uncaches when asked' do + gemspec = Gem::Specification.find_by_name('kramdown') + expect do + pin_cache.uncache_gem(gemspec, out: nil) + end.not_to raise_error + end + end + + context 'with the rebuild flag' do + before do + allow(Solargraph::Yardoc).to receive(:build_docs) + end + + it 'chooses not to use YARD' do + parser_gemspec = Gem::Specification.find_by_name('parser') + pin_cache.cache_gem(gemspec: parser_gemspec, rebuild: true, out: nil) + # if this fails, you may not have run `bundle exec rbs collection update` + expect(Solargraph::Yardoc).not_to have_received(:build_docs).with(any_args) + end + end + + context 'with a stdlib gem' do + let(:gem_name) { 'logger' } + + before do + pin_cache.uncache_gem(Gem::Specification.find_by_name(gem_name), out: nil) + end + + it 'caches' do + yaml_gemspec = Gem::Specification.find_by_name(gem_name) + allow(File).to receive(:write).and_call_original + + pin_cache.cache_gem(gemspec: yaml_gemspec, out: nil) + + # match arguments with regexp using rspec-matchers syntax + expect(File).to have_received(:write).with(%r{combined/.*/logger-.*-stdlib.ser$}, any_args).once + end + end + + context 'with gem packaged with its own RBS' do + let(:gem_name) { 'rubocop-yard' } + + before do + pin_cache.uncache_gem(Gem::Specification.find_by_name(gem_name), out: nil) + end + + it 'caches' do + yaml_gemspec = Gem::Specification.find_by_name(gem_name) + allow(File).to receive(:write).and_call_original + + pin_cache.cache_gem(gemspec: yaml_gemspec, out: nil) + + # match arguments with regexp using rspec-matchers syntax + expect(File).to have_received(:write).with(%r{combined/.*/rubocop-yard-.*-export.ser$}, any_args, + mode: 'wb').once + end + end + end + + describe '#uncache_gem' do + subject(:call) { pin_cache.uncache_gem(gemspec, out: out) } + + let(:out) { StringIO.new } + + before do + allow(FileUtils).to receive(:rm_rf) + end + + context 'with an already cached gem' do + let(:gemspec) { Gem::Specification.find_by_name('backport') } + + it 'deletes files' do + call + + expect(FileUtils).to have_received(:rm_rf).at_least(:once) + end + end + + context 'with a non-existent gem' do + let(:gemspec) { instance_double(Gem::Specification, name: 'nonexistent', version: '0.0.1') } + + it 'does not raise an error' do + expect { call }.not_to raise_error + end + + it 'logs a message' do + call + + expect(out.string).to include('does not exist') + end + + it 'does not delete files' do + call + + expect(FileUtils).not_to have_received(:rm_rf) + end + end + end + + describe '.uncache_by_prefix' do + it 'deletes every file matching the prefix and logs each one' do + Dir.mktmpdir do |dir| + prefix = File.join(dir, 'some-gem-1.0.0') + File.write("#{prefix}-yard.ser", '') + File.write("#{prefix}-rbs.ser", '') + File.write(File.join(dir, 'unrelated-file'), '') + out = StringIO.new + + described_class.uncache_by_prefix(prefix, out: out) + + expect(Dir.glob("#{prefix}*")).to be_empty + expect(File.exist?(File.join(dir, 'unrelated-file'))).to be(true) + expect(out.string).to include('Clearing pin cache in') + end + end + + it 'skips directories matching the prefix glob' do + Dir.mktmpdir do |dir| + prefix = File.join(dir, 'some-gem-1.0.0') + Dir.mkdir("#{prefix}-dir") + + expect { described_class.uncache_by_prefix(prefix) }.not_to raise_error + expect(Dir.exist?("#{prefix}-dir")).to be(true) + end + end + end + + describe '.exist?' do + it 'is true when the joined path is a file' do + Dir.mktmpdir do |dir| + path = File.join(dir, 'cached.ser') + File.write(path, '') + + expect(described_class.exist?(dir, 'cached.ser')).to be(true) + end + end + + it 'is false when the joined path does not exist' do + Dir.mktmpdir do |dir| + expect(described_class.exist?(dir, 'missing.ser')).to be(false) + end + end + end +end diff --git a/spec/rbs_map/core_map_spec.rb b/spec/rbs_map/core_map_spec.rb index 94cd8395b..6f1c48bec 100644 --- a/spec/rbs_map/core_map_spec.rb +++ b/spec/rbs_map/core_map_spec.rb @@ -82,7 +82,7 @@ # correctly. It would be better to test RbsMap or RbsMap::Conversions # with an RBS fixture. core_map = described_class.new - pins = core_map.pins.select { |pin| pin.is_a?(Solargraph::Pin::Reference::Include) && pin.name == 'Enumerable' } + pins = core_map.pins.select { |pin| pin.is_a?(Solargraph::Pin::Reference::Include) && pin.name == '::Enumerable' } expect(pins.map(&:closure).map(&:namespace)).to include('Enumerator') end diff --git a/spec/rbs_map/stdlib_map_spec.rb b/spec/rbs_map/stdlib_map_spec.rb index 4364fcfef..9f76b3d08 100644 --- a/spec/rbs_map/stdlib_map_spec.rb +++ b/spec/rbs_map/stdlib_map_spec.rb @@ -6,7 +6,7 @@ # @todo Unlike the YardMap stdlib, the RBS version reports the correct # return type for Pathname#Join. Delete or modify this test depending # on how StdLibFills will be handled going forward. - rbs_map = Solargraph::RbsMap::StdlibMap.load('pathname') + rbs_map = described_class.load('pathname') pin = rbs_map.path_pin('Pathname#join') expect(pin.signatures.first.return_type.tag).to eq('Pathname') end @@ -25,7 +25,7 @@ it 'processes RBS class variables' do pending 'rbs not in stdlib?' - map = Solargraph::RbsMap::StdlibMap.load('rbs') + map = described_class.load('rbs') store = Solargraph::ApiMap::Store.new(map.pins) class_variable_pins = store.pins_by_class(Solargraph::Pin::ClassVariable) count_pins = class_variable_pins.select do |pin| @@ -38,7 +38,7 @@ it 'processes RBS class instance variables' do pending 'rbs not in stdlib?' - map = Solargraph::RbsMap::StdlibMap.load('rbs') + map = described_class.load('rbs') store = Solargraph::ApiMap::Store.new(map.pins) instance_variable_pins = store.pins_by_class(Solargraph::Pin::InstanceVariable) root_pins = instance_variable_pins.select do |pin| @@ -50,7 +50,7 @@ end it 'processes RBS module aliases' do - map = Solargraph::RbsMap::StdlibMap.load('yaml') + map = described_class.load('yaml') store = Solargraph::ApiMap::Store.new(map.pins) constant_pins = store.get_constants('') yaml_pins = constant_pins.select do |pin| @@ -63,7 +63,7 @@ end it 'pins are marked as coming from RBS parsing' do - map = Solargraph::RbsMap::StdlibMap.load('yaml') + map = described_class.load('yaml') store = Solargraph::ApiMap::Store.new(map.pins) constant_pins = store.get_constants('') pin = constant_pins.first diff --git a/spec/shell_spec.rb b/spec/shell_spec.rb index 3d8a254bf..c910613c2 100644 --- a/spec/shell_spec.rb +++ b/spec/shell_spec.rb @@ -129,6 +129,78 @@ def bundle_exec(*cmd) expect(output).to include("Gem 'solargraph123' not found") end end + + context 'with mocked Workspace' do + let(:workspace) { instance_double(Solargraph::Workspace) } + let(:gemspec) { instance_double(Gem::Specification, name: 'backport') } + + before do + allow(Solargraph::Workspace).to receive(:new).and_return(workspace) + end + + it 'caches all without erroring out' do + allow(workspace).to receive(:cache_all_for_workspace!) + + _output = capture_both { shell.gems } + + expect(workspace).to have_received(:cache_all_for_workspace!) + end + + it 'caches single gem without erroring out' do + allow(workspace).to receive(:find_gem).with('backport').and_return(gemspec) + allow(workspace).to receive(:cache_gem) + + capture_both do + shell.options = { rebuild: false } + shell.gems('backport') + end + + expect(workspace).to have_received(:cache_gem).with(gemspec, out: an_instance_of(StringIO), rebuild: false) + end + + it 'reports a gem not found when find_gem raises Gem::MissingSpecError' do + allow(workspace).to receive(:find_gem) + .and_raise(Gem::MissingSpecError.new('backport', Gem::Requirement.new('>= 0'))) + + output = capture_both { shell.gems('backport') } + + expect(output).to include("Gem 'backport' not found") + end + + it 'reports the failure when find_gem raises Gem::Requirement::BadRequirementError' do + allow(workspace).to receive(:find_gem) + .and_raise(Gem::Requirement::BadRequirementError, 'bad requirement') + + output = capture_both { shell.gems('backport') } + + expect(output).to include("Gem 'backport' failed while loading") + expect(output).to include('bad requirement') + end + + it "caches core pins when name is 'core'" do + allow(Solargraph::PinCache).to receive(:core?).and_return(false) + allow(Solargraph::PinCache).to receive(:cache_core) + + capture_both do + shell.options = { rebuild: false } + shell.gems('core') + end + + expect(Solargraph::PinCache).to have_received(:cache_core).with(out: an_instance_of(StringIO)) + end + + it "rebuilds core pins when name is 'core' and --rebuild is set even if already cached" do + allow(Solargraph::PinCache).to receive(:core?).and_return(true) + allow(Solargraph::PinCache).to receive(:cache_core) + + capture_both do + shell.options = { rebuild: true } + shell.gems('core') + end + + expect(Solargraph::PinCache).to have_received(:cache_core).with(out: an_instance_of(StringIO)) + end + end end describe 'cache' do diff --git a/spec/type_checker/levels/normal_spec.rb b/spec/type_checker/levels/normal_spec.rb index 8dec2892c..21243d161 100644 --- a/spec/type_checker/levels/normal_spec.rb +++ b/spec/type_checker/levels/normal_spec.rb @@ -223,6 +223,9 @@ def bar; end # @todo This test uses kramdown-parser-gfm because it's a gem dependency known to # lack typed methods. A better test wouldn't depend on the state of # vendored code. + workspace = Solargraph::Workspace.new(Dir.pwd) + gemspec = Gem::Specification.find_by_name('kramdown-parser-gfm') + workspace.cache_gem(gemspec) checker = type_checker(%( require 'kramdown-parser-gfm' diff --git a/spec/workspace/gemspecs_resolve_require_spec.rb b/spec/workspace/gemspecs_resolve_require_spec.rb index 8deba9ff8..0764f7ec2 100644 --- a/spec/workspace/gemspecs_resolve_require_spec.rb +++ b/spec/workspace/gemspecs_resolve_require_spec.rb @@ -153,6 +153,31 @@ def configure_bundler_spec stub_value end end + context 'with a require path that does not textually match the gem name' do + # e.g. activesupport ships as 'active_support' - neither + # require.tr('/', '-') nor require.split('/').first can guess + # 'activesupport' from 'active_support', so this can only be + # resolved via Gem::Specification.find_by_path, and only if the + # require path itself (not "lib/#{require}.rb") is passed to it + let(:require) { 'active_support' } + let(:mismatched_spec) { instance_double(Gem::Specification, name: 'activesupport', files: []) } + + before do + allow(Gem::Specification).to receive(:find_by_path).and_call_original + allow(Gem::Specification).to receive(:find_by_path).with(require).and_return(mismatched_spec) + allow(gemspecs).to receive(:all_gemspecs_from_bundle).and_return([mismatched_spec]) + end + + it 'resolves to the right known gem' do + expect(specs.map(&:name)).to eq(['activesupport']) + end + + it 'passes the require path directly to find_by_path, not prefixed with lib/' do + specs + expect(Gem::Specification).to have_received(:find_by_path).with(require) + end + end + context 'with Bundler.require' do let(:require) { 'bundler/require' } diff --git a/spec/workspace_spec.rb b/spec/workspace_spec.rb index ddb5c7f01..427f511b5 100644 --- a/spec/workspace_spec.rb +++ b/spec/workspace_spec.rb @@ -145,4 +145,55 @@ described_class.new('./path', config) end.not_to raise_error end + + describe '#gemfile?' do + it 'returns true when the workspace directory has a Gemfile' do + File.write(File.join(dir_path, 'Gemfile'), "source 'https://rubygems.org'") + + expect(workspace.gemfile?).to be(true) + end + + it 'returns false when the workspace directory has no Gemfile' do + expect(workspace.gemfile?).to be(false) + end + end + + describe '#cache_all_for_workspace!' do + let(:pin_cache) { instance_double(Solargraph::PinCache) } + let(:gemspecs) { instance_double(Solargraph::Workspace::Gemspecs) } + + before do + allow(Solargraph::PinCache).to receive(:cache_core) + allow(Solargraph::PinCache).to receive(:possible_stdlibs) + allow(Solargraph::PinCache).to receive(:new).and_return(pin_cache) + allow(pin_cache).to receive_messages(cache_gem: nil, possible_stdlibs: []) + allow(Solargraph::PinCache).to receive(:cache_all_stdlibs) + allow(Solargraph::Workspace::Gemspecs).to receive(:new).and_return(gemspecs) + gemspec = instance_double(Gem::Specification, name: 'test_gem', version: '1.0.0') + allow(gemspecs).to receive(:all_gemspecs_from_bundle).and_return([gemspec]) + end + + it 'caches core pins' do + allow(Solargraph::PinCache).to receive_messages(core?: false) + allow(pin_cache).to receive_messages(cached?: true, + cache_all_stdlibs: nil) + + workspace.cache_all_for_workspace!(nil, rebuild: false) + + expect(Solargraph::PinCache).to have_received(:cache_core).with(out: nil) + end + + it 'caches gems' do + allow(pin_cache).to receive(:cached?).and_return(false) + + allow(pin_cache).to receive(:cache_all_stdlibs).with(out: nil, rebuild: false) + + allow(Solargraph::PinCache).to receive_messages(core?: true, + possible_stdlibs: []) + + workspace.cache_all_for_workspace!(nil, rebuild: false) + + expect(pin_cache).to have_received(:cache_gem) + end + end end diff --git a/spec/yard_map/mapper_spec.rb b/spec/yard_map/mapper_spec.rb index 9c5decde3..0489d36e3 100644 --- a/spec/yard_map/mapper_spec.rb +++ b/spec/yard_map/mapper_spec.rb @@ -1,5 +1,7 @@ # frozen_string_literal: true +require 'tmpdir' + describe Solargraph::YardMap::Mapper do before :all do # rubocop:disable RSpec/BeforeAfterAll @api_map = Solargraph::ApiMap.load('.') @@ -7,7 +9,7 @@ def pins_with require doc_map = Solargraph::DocMap.new([require], @api_map.workspace, out: nil) - doc_map.cache_all!(nil) + doc_map.cache_doc_map_gems!(nil) doc_map.pins end @@ -108,9 +110,15 @@ def pins_with require it 'adjusts YARD namespaces that conflict with core constants' do gemspec = Gem::Specification.find_by_name('pp') - code_objects = Solargraph::Yardoc.load!(gemspec) - mapper = described_class.new(code_objects) - pins = mapper.map - expect(pins.map(&:path)).to include('RBS::Unnamed::ENVClass#pretty_print') + gem_yardoc_path = Dir.mktmpdir + begin + Solargraph::Yardoc.build_docs(gem_yardoc_path, [], gemspec) + code_objects = Solargraph::Yardoc.load!(gem_yardoc_path) + mapper = described_class.new(code_objects) + pins = mapper.map + expect(pins.map(&:path)).to include('RBS::Unnamed::ENVClass#pretty_print') + ensure + FileUtils.remove_entry_secure(gem_yardoc_path) + end end end diff --git a/spec/yardoc_spec.rb b/spec/yardoc_spec.rb index 5ad0e5805..f57acfb69 100644 --- a/spec/yardoc_spec.rb +++ b/spec/yardoc_spec.rb @@ -4,18 +4,42 @@ require 'open3' describe Solargraph::Yardoc do + around do |testobj| + @tmpdir = Dir.mktmpdir + + testobj.run + ensure + FileUtils.remove_entry(@tmpdir) + end + let(:gem_yardoc_path) do - Solargraph::PinCache.yardoc_path gemspec + File.join(@tmpdir, 'solargraph', 'yardoc', 'test_gem') end before do FileUtils.mkdir_p(gem_yardoc_path) end - describe '#cache' do - let(:api_map) { Solargraph::ApiMap.new } - let(:doc_map) { api_map.doc_map } - let(:gemspec) { Gem::Specification.find_by_path('rubocop') } + describe '#processing?' do + it 'returns true if the yardoc is being processed' do + FileUtils.touch(File.join(gem_yardoc_path, 'processing')) + expect(described_class.processing?(gem_yardoc_path)).to be(true) + end + + it 'returns false if the yardoc is not being processed' do + expect(described_class.processing?(gem_yardoc_path)).to be(false) + end + end + + describe '#load!' do + it 'does not blow up when called on empty directory' do + expect { described_class.load!(gem_yardoc_path) }.not_to raise_error + end + end + + describe '#build_docs' do + let(:workspace) { Solargraph::Workspace.new(Dir.pwd) } + let(:gemspec) { workspace.find_gem('rubocop') } let(:output) { '' } before do @@ -24,6 +48,43 @@ FileUtils.rm_rf(gem_yardoc_path) end + it 'builds docs for a gem' do + described_class.build_docs(gem_yardoc_path, [], gemspec) + expect(File.exist?(File.join(gem_yardoc_path, 'complete'))).to be true + end + + it 'bails quietly if directory given does not exist' do + allow(Dir).to receive(:exist?).and_return(false) + allow(Open3).to receive(:capture2e) + + expect do + described_class.build_docs(gem_yardoc_path, [], gemspec) + end.not_to raise_error + expect(Open3).not_to have_received(:capture2e) + end + + it 'is idempotent' do + described_class.build_docs(gem_yardoc_path, [], gemspec) + described_class.build_docs(gem_yardoc_path, [], gemspec) # second time + expect(File.exist?(File.join(gem_yardoc_path, 'complete'))).to be true + end + + context 'with an error from yard' do + before do + allow(Open3).to receive(:capture2e).and_return([output, result]) + end + + let(:result) { instance_double(Process::Status) } + + it 'does not raise on error from yard' do + allow(result).to receive(:success?).and_return(false) + + expect do + described_class.build_docs(gem_yardoc_path, [], gemspec) + end.not_to raise_error + end + end + context 'when given a relative BUNDLE_GEMFILE path' do around do |example| # turn absolute BUNDLE_GEMFILE path into relative @@ -43,7 +104,7 @@ ['output', instance_double(Process::Status, success?: true)] end - described_class.cache([], gemspec) + described_class.build_docs(gem_yardoc_path, [], gemspec) expect(called_with[0]['BUNDLE_GEMFILE']).to eq(File.absolute_path('Gemfile')) end