Fix line-ending characters

This commit is contained in:
j2blake 2013-08-26 11:49:47 -04:00
parent 22f29cb958
commit 07cf4ad00d
2 changed files with 506 additions and 506 deletions

View file

@ -1,411 +1,411 @@
=begin =begin
-------------------------------------------------------------------------------- --------------------------------------------------------------------------------
Scan the source directory, checking for expected "magic license tags", Scan the source directory, checking for expected "magic license tags",
or or
Copy the source directory, inserting licensing information into the files. Copy the source directory, inserting licensing information into the files.
-------------------------------------------------------------------------------- --------------------------------------------------------------------------------
-------------------------------------------------------------------------------- --------------------------------------------------------------------------------
=end =end
$: << File.dirname(File.expand_path(__FILE__)) $: << File.dirname(File.expand_path(__FILE__))
require 'date' require 'date'
require 'fileutils' require 'fileutils'
require 'pathname' require 'pathname'
require 'property_file_reader' require 'property_file_reader'
require 'licenser_stats' require 'licenser_stats'
class Licenser class Licenser
MAGIC_STRING = '$This file is distributed under the terms of the license in /doc/license.txt$' MAGIC_STRING = '$This file is distributed under the terms of the license in /doc/license.txt$'
# ------------------------------------------------------------------------------------ # ------------------------------------------------------------------------------------
private private
# ------------------------------------------------------------------------------------ # ------------------------------------------------------------------------------------
# #
# Some paths in the properties file, if they are relative, should be relative to the # Some paths in the properties file, if they are relative, should be relative to the
# properties file itself. # properties file itself.
def relative_to_properties(properties, key) def relative_to_properties(properties, key)
path = properties[key] path = properties[key]
base = File.dirname(properties['properties_file_path']) base = File.dirname(properties['properties_file_path'])
return nil if path == nil return nil if path == nil
return File.expand_path(path) if Pathname.new(path).absolute? return File.expand_path(path) if Pathname.new(path).absolute?
return File.expand_path(File.join(base, path)) return File.expand_path(File.join(base, path))
end end
# Some paths in the properties file, if they are relative, should be relative to the # Some paths in the properties file, if they are relative, should be relative to the
# source directory. # source directory.
def relative_to_source(properties, key) def relative_to_source(properties, key)
path = properties[key] path = properties[key]
base = @source_dir ? @source_dir : '' base = @source_dir ? @source_dir : ''
return nil if path == nil return nil if path == nil
return path if Pathname.new(path).absolute? return path if Pathname.new(path).absolute?
return File.expand_path(File.join(base, path)) return File.expand_path(File.join(base, path))
end end
# Confirm that the parameters are reasonable. # Confirm that the parameters are reasonable.
# #
def sanity_checks_on_parameters() def sanity_checks_on_parameters()
# Check that all necessary properties are here. # Check that all necessary properties are here.
raise("Properties file must contain a value for 'scan_only'") if @scan_only_string == nil raise("Properties file must contain a value for 'scan_only'") if @scan_only_string == nil
raise("Properties file must contain a value for 'source_dir'") if @source_dir == nil raise("Properties file must contain a value for 'source_dir'") if @source_dir == nil
raise("Properties file must contain a value for 'known_exceptions'") if @known_exceptions_file == nil raise("Properties file must contain a value for 'known_exceptions'") if @known_exceptions_file == nil
raise("Properties file must contain a value for 'skip_directories'") if @skip_directories_list == nil raise("Properties file must contain a value for 'skip_directories'") if @skip_directories_list == nil
raise("Properties file must contain a value for 'file_matchers'") if @file_match_list == nil raise("Properties file must contain a value for 'file_matchers'") if @file_match_list == nil
raise("Properties file must contain a value for 'report_level'") if @report_level_string == nil raise("Properties file must contain a value for 'report_level'") if @report_level_string == nil
if !File.exist?(@source_dir) if !File.exist?(@source_dir)
raise "Source directory does not exist: #{@source_dir}" raise "Source directory does not exist: #{@source_dir}"
end end
if !File.exist?(@known_exceptions_file) if !File.exist?(@known_exceptions_file)
raise "Known exceptions file does not exist: #{@known_exceptions_file}" raise "Known exceptions file does not exist: #{@known_exceptions_file}"
end end
if !@scan_only if !@scan_only
raise("Properties file must contain a value for 'target_dir'") if @target_dir == nil raise("Properties file must contain a value for 'target_dir'") if @target_dir == nil
raise("Properties file must contain a value for 'license_file'") if @license_file == nil raise("Properties file must contain a value for 'license_file'") if @license_file == nil
if File.exist?(@target_dir) if File.exist?(@target_dir)
raise "Target directory already exists: #{@target_dir}" raise "Target directory already exists: #{@target_dir}"
end end
target_parent = File.dirname(@target_dir) target_parent = File.dirname(@target_dir)
if !File.exist?(target_parent) if !File.exist?(target_parent)
raise "Path to target directory doesn't exist: #{target_parent}" raise "Path to target directory doesn't exist: #{target_parent}"
end end
if !File.exist?(@license_file) if !File.exist?(@license_file)
raise "License file does not exist: #{@license_file}" raise "License file does not exist: #{@license_file}"
end end
end end
end end
# Prepare the license as an array of lines of text, # Prepare the license as an array of lines of text,
# with the current year substituted in for ${year} # with the current year substituted in for ${year}
# #
def prepare_license_text(license_file) def prepare_license_text(license_file)
if (license_file == nil) if (license_file == nil)
return [] return []
end end
year_string = DateTime.now.year.to_s year_string = DateTime.now.year.to_s
text = [] text = []
File.open(license_file) do |file| File.open(license_file) do |file|
file.each do |line| file.each do |line|
text << line.gsub('${year}', year_string) text << line.gsub('${year}', year_string)
end end
end end
return text return text
end end
# The globs in the exceptions file are assumed to be # The globs in the exceptions file are assumed to be
# relative to the source directory. Make them explicitly so. # relative to the source directory. Make them explicitly so.
# #
# Ignore any blank lines or lines that start with a '#' # Ignore any blank lines or lines that start with a '#'
# #
def prepare_exception_globs(exceptions_file, source_dir) def prepare_exception_globs(exceptions_file, source_dir)
source_path = File.expand_path(source_dir) source_path = File.expand_path(source_dir)
globs = [] globs = []
File.open(exceptions_file) do |file| File.open(exceptions_file) do |file|
file.each do |line| file.each do |line|
glob = line.strip glob = line.strip
if (glob.length > 0) && (glob[0..0] != '#') if (glob.length > 0) && (glob[0..0] != '#')
globs << "#{source_path}/#{glob}".gsub('//', '/') globs << "#{source_path}/#{glob}".gsub('//', '/')
end end
end end
end end
return globs return globs
end end
# Recursively scan this directory, and copy if we are not scan-only. # Recursively scan this directory, and copy if we are not scan-only.
# #
def scan_dir(source_dir, target_dir) def scan_dir(source_dir, target_dir)
@stats.enter_directory(source_dir) @stats.enter_directory(source_dir)
Dir.mkdir(File.expand_path(target_dir, @target_dir)) if !@scan_only Dir.mkdir(File.expand_path(target_dir, @target_dir)) if !@scan_only
Dir.foreach(File.join(@source_dir, source_dir)) do |filename| Dir.foreach(File.join(@source_dir, source_dir)) do |filename|
source_path_relative = File.join(source_dir, filename) source_path_relative = File.join(source_dir, filename)
source_path = File.join(@source_dir, source_path_relative) source_path = File.join(@source_dir, source_path_relative)
target_path_relative = File.join(target_dir, filename) target_path_relative = File.join(target_dir, filename)
target_path = File.join(@target_dir, target_path_relative) target_path = File.join(@target_dir, target_path_relative)
# What kind of beast is this? # What kind of beast is this?
if filename == '.' || filename == '..' if filename == '.' || filename == '..'
is_skipped_directory = true is_skipped_directory = true
else else
if File.directory?(source_path) if File.directory?(source_path)
if (path_matches_skipped?(source_path_relative)) if (path_matches_skipped?(source_path_relative))
is_skipped_directory = true is_skipped_directory = true
else else
is_directory = true is_directory = true
end end
else else
if filename_matches_pattern?(filename) if filename_matches_pattern?(filename)
if path_matches_exception?(source_path_relative) if path_matches_exception?(source_path_relative)
is_exception = true is_exception = true
else else
is_match = true is_match = true
end end
else else
is_ignored = true is_ignored = true
end end
end end
end end
if is_skipped_directory if is_skipped_directory
# do nothing # do nothing
elsif is_directory elsif is_directory
scan_dir(source_path_relative, target_path_relative) scan_dir(source_path_relative, target_path_relative)
elsif is_match elsif is_match
if @scan_only if @scan_only
@stats.record_scan_matching(filename) @stats.record_scan_matching(filename)
scan_file(source_path, filename) scan_file(source_path, filename)
else else
@stats.record_copy_matching(filename) @stats.record_copy_matching(filename)
copy_file_with_license(source_path, target_path, filename) copy_file_with_license(source_path, target_path, filename)
end end
elsif is_exception elsif is_exception
@stats.record_known_exception(filename) @stats.record_known_exception(filename)
if @scan_only if @scan_only
# do nothing # do nothing
else else
copy_file_without_license(source_path, target_path) copy_file_without_license(source_path, target_path)
end end
else # not a match else # not a match
if @scan_only if @scan_only
@stats.record_scan_non_matching(filename) @stats.record_scan_non_matching(filename)
else else
@stats.record_copy_non_matching(filename) @stats.record_copy_non_matching(filename)
copy_file_without_license(source_path, target_path) copy_file_without_license(source_path, target_path)
end end
end end
end end
end end
# Is this directory one of the skipped? # Is this directory one of the skipped?
# #
def path_matches_skipped?(relative_path) def path_matches_skipped?(relative_path)
@skip_directories.each do |glob| @skip_directories.each do |glob|
return true if File.fnmatch(glob, relative_path) return true if File.fnmatch(glob, relative_path)
end end
return false return false
end end
# Does this file path match any of the exceptions? # Does this file path match any of the exceptions?
# #
def path_matches_exception?(relative_path) def path_matches_exception?(relative_path)
path = File.expand_path(File.join(@source_dir, relative_path)) path = File.expand_path(File.join(@source_dir, relative_path))
@known_exceptions.each do |pattern| @known_exceptions.each do |pattern|
return true if File.fnmatch(pattern, path) return true if File.fnmatch(pattern, path)
end end
return false return false
end end
# Does this filename match any of the patterns? # Does this filename match any of the patterns?
# #
def filename_matches_pattern?(filename) def filename_matches_pattern?(filename)
@file_matchers.each do |pattern| @file_matchers.each do |pattern|
return true if File.fnmatch(pattern, filename) return true if File.fnmatch(pattern, filename)
end end
return false return false
end end
# This file would be eligible for licensing if we weren't in scan-only mode. # This file would be eligible for licensing if we weren't in scan-only mode.
# #
def scan_file(source_path, filename) def scan_file(source_path, filename)
found = 0 found = 0
File.open(source_path) do |source_file| File.open(source_path) do |source_file|
source_file.each do |line| source_file.each do |line|
if line.include?(MAGIC_STRING) if line.include?(MAGIC_STRING)
found += 1 found += 1
end end
end end
end end
if found == 0 if found == 0
@stats.record_no_tag(filename, source_path) @stats.record_no_tag(filename, source_path)
elsif found == 1 elsif found == 1
@stats.record_tag(filename) @stats.record_tag(filename)
else else
raise("File contains #{found} license lines: #{source_path}") raise("File contains #{found} license lines: #{source_path}")
end end
end end
# This file matches at least one of the file-matching strings, and does not # This file matches at least one of the file-matching strings, and does not
# match any exceptions. Replace the magic string with the license text. # match any exceptions. Replace the magic string with the license text.
# #
def copy_file_with_license(source_path, target_path, filename) def copy_file_with_license(source_path, target_path, filename)
found = 0 found = 0
File.open(source_path) do |source_file| File.open(source_path) do |source_file|
File.open(target_path, "w") do |target_file| File.open(target_path, "w") do |target_file|
source_file.each do |line| source_file.each do |line|
if line.include?(MAGIC_STRING) if line.include?(MAGIC_STRING)
found += 1 found += 1
insert_license_text(target_file, line) insert_license_text(target_file, line)
else else
target_file.print line target_file.print line
end end
end end
end end
end end
if found == 0 if found == 0
@stats.record_no_tag(filename, source_path) @stats.record_no_tag(filename, source_path)
elsif found == 1 elsif found == 1
@stats.record_tag(filename) @stats.record_tag(filename)
else else
raise("File contains #{found} license lines: #{source_path}") raise("File contains #{found} license lines: #{source_path}")
end end
end end
# Figure out the comment characters and write the license text to the file. # Figure out the comment characters and write the license text to the file.
# #
def insert_license_text(target_file, line) def insert_license_text(target_file, line)
ends = line.split(MAGIC_STRING) ends = line.split(MAGIC_STRING)
if ends.size != 2 if ends.size != 2
raise ("Can't parse this license line: #{line}") raise ("Can't parse this license line: #{line}")
end end
target_file.print "#{ends[0].strip}\n" target_file.print "#{ends[0].strip}\n"
@license_text.each do |text| @license_text.each do |text|
target_file.print "#{text.rstrip}\n" target_file.print "#{text.rstrip}\n"
end end
target_file.print "#{ends[1].strip}\n" target_file.print "#{ends[1].strip}\n"
end end
# This file either doesn't match any of the file-matching strings, or # This file either doesn't match any of the file-matching strings, or
# matches an exception # matches an exception
# #
def copy_file_without_license(source_path, target_path) def copy_file_without_license(source_path, target_path)
FileUtils.cp(source_path, target_path) FileUtils.cp(source_path, target_path)
end end
# ------------------------------------------------------------------------------------ # ------------------------------------------------------------------------------------
public public
# ------------------------------------------------------------------------------------ # ------------------------------------------------------------------------------------
# Setup and get ready to process. # Setup and get ready to process.
# #
# * properties is a map of keys to values, probably parsed from a properties file. # * properties is a map of keys to values, probably parsed from a properties file.
# #
def initialize(properties) def initialize(properties)
@scan_only_string = properties['scan_only'] @scan_only_string = properties['scan_only']
@scan_only = 'false' != @scan_only_string @scan_only = 'false' != @scan_only_string
@file_match_list = properties['file_matchers'] @file_match_list = properties['file_matchers']
@skip_directories_list = properties['skip_directories'] @skip_directories_list = properties['skip_directories']
@report_level_string = properties['report_level'] @report_level_string = properties['report_level']
# These properties contain paths, and if they are relative paths, they # These properties contain paths, and if they are relative paths, they
# should be relative to the properties file itself. # should be relative to the properties file itself.
@source_dir = relative_to_properties(properties, 'source_dir') @source_dir = relative_to_properties(properties, 'source_dir')
@target_dir = relative_to_properties(properties, 'target_dir') @target_dir = relative_to_properties(properties, 'target_dir')
# These properties contain paths, and if they are relative paths, they # These properties contain paths, and if they are relative paths, they
# should be relative to the source_directory. # should be relative to the source_directory.
@license_file = relative_to_source(properties, 'license_file') @license_file = relative_to_source(properties, 'license_file')
@known_exceptions_file = relative_to_source(properties, 'known_exceptions') @known_exceptions_file = relative_to_source(properties, 'known_exceptions')
sanity_checks_on_parameters() sanity_checks_on_parameters()
@full_report = @report_level_string === 'full' @full_report = @report_level_string === 'full'
@short_report = @report_level_string === 'short' @short_report = @report_level_string === 'short'
@file_matchers = @file_match_list.strip.split(/,\s*/) @file_matchers = @file_match_list.strip.split(/,\s*/)
@skip_directories = @skip_directories_list.strip.split(/,\s*/) @skip_directories = @skip_directories_list.strip.split(/,\s*/)
@license_text = prepare_license_text(@license_file) @license_text = prepare_license_text(@license_file)
@known_exceptions = prepare_exception_globs(@known_exceptions_file, @source_dir) @known_exceptions = prepare_exception_globs(@known_exceptions_file, @source_dir)
@stats = LicenserStats.new(@source_dir, @file_matchers, @full_report) @stats = LicenserStats.new(@source_dir, @file_matchers, @full_report)
end end
# Start the recursive scanning (and copying). # Start the recursive scanning (and copying).
def process() def process()
scan_dir('.', '.') scan_dir('.', '.')
end end
# Report the summary statistics # Report the summary statistics
def report(properties) def report(properties)
verb = @scan_only ? "scanned" : "copied" verb = @scan_only ? "scanned" : "copied"
if (@short_report) if (@short_report)
subs = 0 subs = 0
@stats.substitutions.each {|line| subs += line[1] } @stats.substitutions.each {|line| subs += line[1] }
known = 0 known = 0
@stats.known_exceptions.each {|line| known += line[1] } @stats.known_exceptions.each {|line| known += line[1] }
missing = 0 missing = 0
@stats.missing_tags.each {|line| missing += line[1] } @stats.missing_tags.each {|line| missing += line[1] }
puts "Licenser: #{verb} #{@stats.file_count} files in #{@stats.dir_count} directories." puts "Licenser: #{verb} #{@stats.file_count} files in #{@stats.dir_count} directories."
printf(" Substitutions: %5d\n", subs) printf(" Substitutions: %5d\n", subs)
printf(" Known exceptions: %5d\n", known) printf(" Known exceptions: %5d\n", known)
printf(" Missing tags: %5d\n", missing) printf(" Missing tags: %5d\n", missing)
else else
puts "Licenser: run completed at #{DateTime.now.strftime("%H:%M:%S on %b %d, %Y")}" puts "Licenser: run completed at #{DateTime.now.strftime("%H:%M:%S on %b %d, %Y")}"
puts " #{verb} #{@stats.file_count} files in #{@stats.dir_count} directories." puts " #{verb} #{@stats.file_count} files in #{@stats.dir_count} directories."
puts puts
puts 'Substitutions' puts 'Substitutions'
@stats.substitutions.sort.each do |line| @stats.substitutions.sort.each do |line|
printf("%5d %s\n", line[1], line[0]) printf("%5d %s\n", line[1], line[0])
end end
puts puts
puts 'Known non-licensed files' puts 'Known non-licensed files'
@stats.known_exceptions.sort.each do |line| @stats.known_exceptions.sort.each do |line|
printf("%5d %s\n", line[1], line[0]) printf("%5d %s\n", line[1], line[0])
end end
puts puts
puts 'Missing tags' puts 'Missing tags'
@stats.missing_tags.sort.each do |line| @stats.missing_tags.sort.each do |line|
printf("%5d %s\n", line[1], line[0]) printf("%5d %s\n", line[1], line[0])
end end
puts puts
puts 'properties:' puts 'properties:'
properties.each do |key, value| properties.each do |key, value|
puts " #{key} = #{value}" puts " #{key} = #{value}"
end end
end end
end end
# Were we successful or not? # Were we successful or not?
def success? def success?
return @stats.missing_tags.empty? return @stats.missing_tags.empty?
end end
end end
# #
# #
# ------------------------------------------------------------------------------------ # ------------------------------------------------------------------------------------
# Standalone calling. # Standalone calling.
# #
# Do this if this program was called from the command line. That is, if the command # Do this if this program was called from the command line. That is, if the command
# expands to the path of this file. # expands to the path of this file.
# ------------------------------------------------------------------------------------ # ------------------------------------------------------------------------------------
# #
if File.expand_path($0) == File.expand_path(__FILE__) if File.expand_path($0) == File.expand_path(__FILE__)
if ARGV.length == 0 if ARGV.length == 0
raise("No arguments - usage is: ruby licenser.rb <properties_file>") raise("No arguments - usage is: ruby licenser.rb <properties_file>")
end end
if !File.file?(ARGV[0]) if !File.file?(ARGV[0])
raise "File does not exist: '#{ARGV[0]}'." raise "File does not exist: '#{ARGV[0]}'."
end end
properties = PropertyFileReader.read(ARGV[0]) properties = PropertyFileReader.read(ARGV[0])
l = Licenser.new(properties) l = Licenser.new(properties)
l.process l.process
l.report(properties) l.report(properties)
if l.success? if l.success?
puts "Licenser was successful." puts "Licenser was successful."
exit 0 exit 0
else else
puts "Licenser found problems." puts "Licenser found problems."
exit 1 exit 1
end end
end end

View file

@ -1,95 +1,95 @@
=begin =begin
-------------------------------------------------------------------------------- --------------------------------------------------------------------------------
Collect the statistics of a licenser run. Collect the statistics of a licenser run.
-------------------------------------------------------------------------------- --------------------------------------------------------------------------------
=end =end
class LicenserStats class LicenserStats
attr_reader :substitutions attr_reader :substitutions
attr_reader :missing_tags attr_reader :missing_tags
attr_reader :known_exceptions attr_reader :known_exceptions
attr_reader :file_count attr_reader :file_count
attr_reader :dir_count attr_reader :dir_count
# ------------------------------------------------------------------------------------ # ------------------------------------------------------------------------------------
private private
# ------------------------------------------------------------------------------------ # ------------------------------------------------------------------------------------
# #
def which_match(filename) def which_match(filename)
@file_matchers.each do |matcher| @file_matchers.each do |matcher|
return matcher if File.fnmatch(matcher, filename) return matcher if File.fnmatch(matcher, filename)
end end
raise("filename matches no matchers!: #{filename}") raise("filename matches no matchers!: #{filename}")
end end
# ------------------------------------------------------------------------------------ # ------------------------------------------------------------------------------------
public public
# ------------------------------------------------------------------------------------ # ------------------------------------------------------------------------------------
def initialize(root_dir, file_matchers, full) def initialize(root_dir, file_matchers, full)
@root_dir = "#{root_dir}/".gsub('//', '/') @root_dir = "#{root_dir}/".gsub('//', '/')
@file_matchers = file_matchers @file_matchers = file_matchers
@full = full @full = full
# keep track of how many substitutions for all file types # keep track of how many substitutions for all file types
@substitutions = Hash.new() @substitutions = Hash.new()
file_matchers.each do |matcher| file_matchers.each do |matcher|
@substitutions[matcher] = 0 @substitutions[matcher] = 0
end end
# keep track of missing tags, only in file types that have missing tags # keep track of missing tags, only in file types that have missing tags
@missing_tags = Hash.new(0) @missing_tags = Hash.new(0)
# keep track of how many known non-licensed files we encounter, and what types. # keep track of how many known non-licensed files we encounter, and what types.
@known_exceptions = Hash.new(0) @known_exceptions = Hash.new(0)
# keep track of how many files are copied # keep track of how many files are copied
@file_count = 0 @file_count = 0
#keep track of how many directories are copied #keep track of how many directories are copied
@dir_count = 0 @dir_count = 0
end end
def enter_directory(path) def enter_directory(path)
@dir_count += 1 @dir_count += 1
puts "Entering directory: #{path}" if @full puts "Entering directory: #{path}" if @full
end end
def record_scan_non_matching(filename) def record_scan_non_matching(filename)
@file_count += 1 @file_count += 1
puts " Scan without mods: #{filename}" if @full puts " Scan without mods: #{filename}" if @full
end end
def record_copy_non_matching(filename) def record_copy_non_matching(filename)
@file_count += 1 @file_count += 1
puts " Copy without mods: #{filename}" if @full puts " Copy without mods: #{filename}" if @full
end end
def record_scan_matching(filename) def record_scan_matching(filename)
@file_count += 1 @file_count += 1
puts " Scan with mods: #{filename}" if @full puts " Scan with mods: #{filename}" if @full
end end
def record_copy_matching(filename) def record_copy_matching(filename)
@file_count += 1 @file_count += 1
puts " Copy with mods: #{filename}" if @full puts " Copy with mods: #{filename}" if @full
end end
def record_known_exception(filename) def record_known_exception(filename)
@file_count += 1 @file_count += 1
puts " Known exception: #{filename}" if @full puts " Known exception: #{filename}" if @full
@known_exceptions[which_match(filename)] += 1 @known_exceptions[which_match(filename)] += 1
end end
def record_tag(filename) def record_tag(filename)
puts " Substituted license text into #{filename}" if @full puts " Substituted license text into #{filename}" if @full
@substitutions[which_match(filename)] += 1 @substitutions[which_match(filename)] += 1
end end
def record_no_tag(filename, source_path) def record_no_tag(filename, source_path)
puts "ERROR: Found no license tag in #{source_path.sub(@root_dir, '')}" puts "ERROR: Found no license tag in #{source_path.sub(@root_dir, '')}"
@missing_tags[which_match(filename)] += 1 @missing_tags[which_match(filename)] += 1
end end
end end