NIHVIVO-241 Move the licenser script into Vitro core, with some re-writing to use relative paths and such, and changes to the build scripts to support it.

This commit is contained in:
jeb228 2010-04-15 20:20:35 +00:00
parent 40990255c7
commit 5b77126d36
5 changed files with 45 additions and 628 deletions

View file

@ -41,6 +41,11 @@
-->
<property name="skip.core.themes" value="true" />
<!--
Tell the licesner where to find its properties for this product.
-->
<property name="licenser.product.properties.file" location="config/licenser/licenser.properties" />
<!--
Now get the standard product-build file.
@ -91,39 +96,4 @@
<arg value="${acceptance.properties.file}" />
</exec>
</target>
<!-- =================================
target: licenser
Hudson (continuous integration)
uses this to check that all
appropriate files have license tags.
================================= -->
<target name="licenser" description="--> Check licensing tags">
<property name="licenser.script.directory" location="utilities/release-tools" />
<property name="licenser.properties.file"
location="${licenser.script.directory}/licenser.properties" />
<fail message="You must create a &quot;${licenser.properties.file}&quot; file.">
<condition>
<not>
<available file="${licenser.properties.file}" />
</not>
</condition>
</fail>
<exec executable="ruby" dir="${licenser.script.directory}" failonerror="true">
<arg value="licenser.rb" />
<arg value="${licenser.properties.file}" />
<redirector outputproperty="licenser.test.output" alwayslog="true" />
</exec>
<fail message="Errors were detected in the console output from the licenser.">
<condition>
<contains string="${licenser.test.output}"
substring="WARN: Found no license tag"
casesensitive="false" />
</condition>
</fail>
</target>
</project>

View file

@ -0,0 +1,20 @@
#
# A list of files and directories that are known exceptions to the
# license-insertion process.
#
# Files will only be altered if they contain a "magic" license place-holder,
# but if they match one of the file-matchers and don't contain a place-holder,
# the process will write a warning.
#
# File-matchers are:
# '*.java', '*.jsp', '*.tld', '*.xsl', '*.xslt', '*.css', '*.js', 'build.xml'
#
# Known exceptions listed here produce no warnings.
#
# PROBLEM: It appears that these files require a LICENSE file to accompany them.
themes/vivo-basic/css/blueprint/grid.css
themes/vivo-basic/css/blueprint/ie.css
# PROBLEM: This is from www.involutionstudios.com, not blueprint. What are the license restrictions?
themes/vivo-basic/css/blueprint/liquid.css

View file

@ -1,25 +1,37 @@
# --------------------------------------------------------------------------
# Properties for running the licenser utility.
# Properties for running the licenser utility in VIVO.
# --------------------------------------------------------------------------
# The path to the top level directory to be scanned or copied
# (if relative, then relative to this file)
source_dir = ../../
# The path to the top level directory to copy into (ignored if only scanning)
target_dir =
# (if relative, then relative to this file)
target_dir =
# A list of filename globs that match the files we want to license,
# delimited by commas with optional white-space.
file_matchers = *.java, *.jsp, *.tld, *.xsl, *.xslt, *.css, *.js, build.xml
# The path to the text of the license agreement (ignored if only scanning)
# If the agreement contains a ${year} token, the current year will be substituted.
license_file = ../../doc/license.txt
# "globs" that describe paths that we won't follow for scanning OR FOR COPYING.
# (relative to the source_dir)
skip_directories = ./bin, ./.svn, ./**/.svn, ./.build, ./vitro-core
# The path to a file containing filename/path globs that match the files that
# we know should have no license tags in them.
# The file contains one glob per line; blank lines and comments ("#") are ignored.
known_exceptions = ./known_exceptions.txt
# (if relative, then relative to the source directory)
known_exceptions = config/licenser/known_exceptions.txt
# if true, we give a full log instead of just a summary.
full_report = false
# The path to the text of the license agreement (ignored if only scanning)
# If the agreement contains a ${year} token, the current year will be substituted.
# (if relative, then relative to the source directory)
license_file = doc/license.txt
# Set to 'full' for a full report, 'short' for a brief statment, or to anything
# else for a medium-length summary.
report_level = short
# if true, we are just scanning, not copying.
scan_only = true

View file

@ -1,137 +0,0 @@
#
# A list of files and directories that are known exceptions to the
# license-insertion process.
#
# Files will only be altered if they contain a "magic" license place-holder,
# but if they match one of the file-matchers and don't contain a place-holder,
# the process will write a warning.
#
# File-matchers are:
# '*.java', '*.jsp', '*.tld', '*.xsl', '*.xslt', '*.css', '*.js', 'build.xml'
#
# Known exceptions listed here produce no warnings.
#
# The "build directory"
.build/**/*
# ContentType from Google gdata API, Apache 2.0 license included in file.
vitro-core/webapp/src/edu/cornell/mannlib/vitro/webapp/web/ContentType.java
# The JSON Java source files from json.org
vitro-core/webapp/src/org/json/*
# TinyMCE from tinymce.moxiecode.com
vitro-core/webapp/web/js/tiny_mce/*
vitro-core/webapp/web/js/tiny_mce/**/*
# From apache.org under the Apache license.
vitro-core/webapp/web/admin/axis/SOAPMonitorApplet.java
vitro-core/services/additions/admin/axis/SOAPMonitorApplet.java
# JQuery from jquery.com
vitro-core/webapp/web/js/jquery.js
vitro-core/webapp/web/js/jquery_plugins/*
# From pajhome.org.uk
vitro-core/webapp/web/js/md5.js
# From dojotoolkit.org
vitro-core/webapp/web/dojo.js
vitro-core/webapp/web/src/AdapterRegistry.js
vitro-core/webapp/web/src/animation/*
vitro-core/webapp/web/src/animation.js
vitro-core/webapp/web/src/behavior.js
vitro-core/webapp/web/src/bootstrap1.js
vitro-core/webapp/web/src/bootstrap2.js
vitro-core/webapp/web/src/browser_debug.js
vitro-core/webapp/web/src/collections/*
vitro-core/webapp/web/src/compat/*
vitro-core/webapp/web/src/crypto/*
vitro-core/webapp/web/src/crypto.js
vitro-core/webapp/web/src/data/*
vitro-core/webapp/web/src/data.js
vitro-core/webapp/web/src/date.js
vitro-core/webapp/web/src/debug/*
vitro-core/webapp/web/src/debug.js
vitro-core/webapp/web/src/Deferred.js
vitro-core/webapp/web/src/dnd/*
vitro-core/webapp/web/src/doc.js
vitro-core/webapp/web/src/dom.js
vitro-core/webapp/web/src/event/*
vitro-core/webapp/web/src/event.js
vitro-core/webapp/web/src/experimental.js
vitro-core/webapp/web/src/flash.js
vitro-core/webapp/web/src/fx/*
vitro-core/webapp/web/src/graphics/*
vitro-core/webapp/web/src/hostenv_adobesvg.js
vitro-core/webapp/web/src/hostenv_browser.js
vitro-core/webapp/web/src/hostenv_dashboard.js
vitro-core/webapp/web/src/hostenv_jsc.js
vitro-core/webapp/web/src/hostenv_rhino.js
vitro-core/webapp/web/src/hostenv_spidermonkey.js
vitro-core/webapp/web/src/hostenv_svg.js
vitro-core/webapp/web/src/hostenv_wsh.js
vitro-core/webapp/web/src/html/*
vitro-core/webapp/web/src/html.js
vitro-core/webapp/web/src/i18n/*
vitro-core/webapp/web/src/iCalendar.js
vitro-core/webapp/web/src/io/*
vitro-core/webapp/web/src/io.js
vitro-core/webapp/web/src/json.js
vitro-core/webapp/web/src/lang/*
vitro-core/webapp/web/src/lang.js
vitro-core/webapp/web/src/lfx/*
vitro-core/webapp/web/src/loader.js
vitro-core/webapp/web/src/loader_xd.js
vitro-core/webapp/web/src/logging/*
vitro-core/webapp/web/src/math/*
vitro-core/webapp/web/src/math.js
vitro-core/webapp/web/src/profile.js
vitro-core/webapp/web/src/reflect/*
vitro-core/webapp/web/src/regexp.js
vitro-core/webapp/web/src/rpc/*
vitro-core/webapp/web/src/selection/*
vitro-core/webapp/web/src/storage/*
vitro-core/webapp/web/src/storage.js
vitro-core/webapp/web/src/string/*
vitro-core/webapp/web/src/string.js
vitro-core/webapp/web/src/style.js
vitro-core/webapp/web/src/svg.js
vitro-core/webapp/web/src/text/*
vitro-core/webapp/web/src/undo/*
vitro-core/webapp/web/src/uri/*
vitro-core/webapp/web/src/uuid/*
vitro-core/webapp/web/src/validate/*
vitro-core/webapp/web/src/validate.js
vitro-core/webapp/web/src/widget/*
vitro-core/webapp/web/src/widget/**/*
vitro-core/webapp/web/src/xml/*
# PROBLEM? Third-party tlds - where do they come from?
vitro-core/webapp/config/tlds/c.tld
vitro-core/webapp/config/tlds/fn.tld
vitro-core/webapp/config/tlds/sparqltag.tld
vitro-core/webapp/config/tlds/taglibs-mailer.tld
vitro-core/webapp/config/tlds/taglibs-random.tld
vitro-core/webapp/config/tlds/taglibs-string.tld
# PROBLEM: It appears that these files require a LICENSE file to accompany them.
themes/vivo-basic/css/blueprint/grid.css
themes/vivo-basic/css/blueprint/ie.css
vitro-core/webapp/web/themes/enhanced/css/blueprint/grid.css
vitro-core/webapp/web/themes/enhanced/css/blueprint/ie.css
# PROBLEM: This is from www.involutionstudios.com, not blueprint. What are the license restrictions?
themes/vivo-basic/css/blueprint/liquid.css
vitro-core/webapp/web/themes/enhanced/css/blueprint/liquid.css
# PROBLEM? from simon.incutio.com, but what are the license restrictions?
vitro-core/webapp/web/js/betterDateInput.js
# PROBLEM? from wrox.com, but what are the license restrictions?
vitro-core/webapp/web/js/detect.js
# PROBLEM? perhaps from www.dannyg.com, but what are the license restrictions?
vitro-core/webapp/web/toggle.js
vitro-core/webapp/web/js/toggle.js

View file

@ -1,448 +0,0 @@
=begin
--------------------------------------------------------------------------------
Create a copy of the source files, with licensing information inserted.
--------------------------------------------------------------------------------
2010-01-26 initial version J.Blake
--------------------------------------------------------------------------------
=end
require 'date'
require 'fileutils'
require File.expand_path('acceptance-tests/script/property_file_reader', File.dirname(File.dirname(File.expand_path(__FILE__))))
class LicenserStats
attr_reader :substitutions
attr_reader :missing_tags
attr_reader :known_exceptions
attr_reader :file_count
attr_reader :dir_count
# ------------------------------------------------------------------------------------
private
# ------------------------------------------------------------------------------------
#
def which_match(filename)
@file_matchers.each do |matcher|
return matcher if File.fnmatch(matcher, filename)
end
raise("filename matches no matchers!: #{filename}")
end
# ------------------------------------------------------------------------------------
public
# ------------------------------------------------------------------------------------
def initialize(root_dir, file_matchers, full)
@root_dir = "#{root_dir}/".gsub('//', '/')
@file_matchers = file_matchers
@full = full
# keep track of how many substitutions for all file types
@substitutions = Hash.new()
file_matchers.each do |matcher|
@substitutions[matcher] = 0
end
# keep track of missing tags, only in file types that have missing tags
@missing_tags = Hash.new(0)
# keep track of how many known non-licensed files we encounter, and what types.
@known_exceptions = Hash.new(0)
# keep track of how many files are copied
@file_count = 0
#keep track of how many directories are copied
@dir_count = 0
end
def enter_directory(path)
@dir_count += 1
puts "Entering directory: #{path}" if @full
end
def record_scan_non_matching(filename)
@file_count += 1
puts " Scan without mods: #{filename}" if @full
end
def record_copy_non_matching(filename)
@file_count += 1
puts " Copy without mods: #{filename}" if @full
end
def record_scan_matching(filename)
@file_count += 1
puts " Scan with mods: #{filename}" if @full
end
def record_copy_matching(filename)
@file_count += 1
puts " Copy with mods: #{filename}" if @full
end
def record_known_exception(filename)
@file_count += 1
puts " Known exception: #{filename}" if @full
@known_exceptions[which_match(filename)] += 1
end
def record_tag(filename)
puts " Substituted license text into #{filename}" if @full
@substitutions[which_match(filename)] += 1
end
def record_no_tag(filename, source_path)
puts "WARN: Found no license tag in #{source_path.sub(@root_dir, '')}"
@missing_tags[which_match(filename)] += 1
end
end
class Licenser
MAGIC_STRING = '$This file is distributed under the terms of the license in /doc/license.txt$'
# ------------------------------------------------------------------------------------
private
# ------------------------------------------------------------------------------------
#
# Confirm that the parameters are reasonable.
#
def sanity_checks_on_parameters()
# Check that all necessary properties are here.
raise("Properties file must contain a value for 'source_dir'") if @source_dir == nil
raise("Properties file must contain a value for 'known_exceptions'") if @known_exceptions_file == nil
raise("Properties file must contain a value for 'file_matchers'") if @file_match_list == nil
raise("Properties file must contain a value for 'full_report'") if @full_report_string == nil
if !File.exist?(@source_dir)
raise "Source directory does not exist: #{@source_dir}"
end
if !File.exist?(@known_exceptions_file)
raise "Known exceptions file does not exist: #{@known_exceptions_file}"
end
if !@scan_only
raise("Properties file must contain a value for 'target_dir'") if @target_dir == nil
raise("Properties file must contain a value for 'license_file'") if @license_file == nil
if File.exist?(@target_dir)
raise "Target directory already exists: #{@target_dir}"
end
target_parent = File.dirname(@target_dir)
if !File.exist?(target_parent)
raise "Path to target directory doesn't exist: #{target_parent}"
end
if !File.exist?(@license_file)
raise "License file does not exist: #{@license_file}"
end
end
end
#
# Prepare the license as an array of lines of text,
# with the current year substituted in for ${year}
#
def prepare_license_text(license_file)
if (license_file == nil)
return []
end
year_string = DateTime.now.year.to_s
text = []
File.open(license_file) do |file|
file.each do |line|
text << line.gsub('${year}', year_string)
end
end
return text
end
# The globs in the exceptions file are assumed to be
# relative to the source directory. Make them explicitly so.
#
# Ignore any blank lines or lines that start with a '#'
#
def prepare_exception_globs(exceptions_file, source_dir)
source_path = File.expand_path(source_dir)
globs = []
File.open(exceptions_file) do |file|
file.each do |line|
glob = line.strip
if (glob.length > 0) && (glob[0..0] != '#')
globs << "#{source_path}/#{glob}".gsub('//', '/')
end
end
end
return globs
end
# Recursively scan this directory, and copy if we are not scan-only.
#
def scan_dir(source_dir, target_dir)
@stats.enter_directory(source_dir)
Dir.mkdir(target_dir) if !@scan_only
Dir.foreach(source_dir) do |filename|
source_path = "#{source_dir}/#{filename}"
target_path = "#{target_dir}/#{filename}"
# What kind of beast is this?
if filename == '.' || filename == '..'
is_skipped_directory = true
else
if File.directory?(source_path)
is_directory = true
else
if filename_matches_pattern?(filename)
if path_matches_exception?(source_path)
is_exception = true
else
is_match = true
end
else
is_ignored = true
end
end
end
if is_skipped_directory
# do nothing
elsif is_directory
scan_dir(source_path, target_path)
elsif is_match
if @scan_only
@stats.record_scan_matching(filename)
scan_file(source_path, filename)
else
@stats.record_copy_matching(filename)
copy_file_with_license(source_path, target_path, filename)
end
elsif is_exception
@stats.record_known_exception(filename)
if @scan_only
# do nothing
else
copy_file_without_license(source_path, target_path)
end
else # not a match
if @scan_only
@stats.record_scan_non_matching(filename)
else
@stats.record_copy_non_matching(filename)
copy_file_without_license(source_path, target_path)
end
end
end
end
# Does this file path match any of the exceptions?
#
def path_matches_exception?(path)
path = File.expand_path(path)
@known_exceptions.each do |pattern|
return true if File.fnmatch(pattern, path)
end
return false
end
# Does this filename match any of the patterns?
#
def filename_matches_pattern?(filename)
@file_matchers.each do |pattern|
return true if File.fnmatch(pattern, filename)
end
return false
end
# This file would be eligible for licensing if we weren't in scan-only mode.
#
def scan_file(source_path, filename)
found = 0
File.open(source_path) do |source_file|
source_file.each do |line|
if line.include?(MAGIC_STRING)
found += 1
end
end
end
if found == 0
@stats.record_no_tag(filename, source_path)
elsif found == 1
@stats.record_tag(filename)
else
raise("File contains #{found} license lines: #{source_path}")
end
end
# This file matches at least one of the file-matching strings, and does not
# match any exceptions. Replace the magic string with the license text.
#
def copy_file_with_license(source_path, target_path, filename)
found = 0
File.open(source_path) do |source_file|
File.open(target_path, "w") do |target_file|
source_file.each do |line|
if line.include?(MAGIC_STRING)
found += 1
insert_license_text(target_file, line)
else
target_file.print line
end
end
end
end
if found == 0
@stats.record_no_tag(filename, source_path)
elsif found == 1
@stats.record_tag(filename)
else
raise("File contains #{found} license lines: #{source_path}")
end
end
# Figure out the comment characters and write the license text to the file.
#
def insert_license_text(target_file, line)
ends = line.split(MAGIC_STRING)
if ends.size != 2
raise ("Can't parse this license line: #{line}")
end
target_file.print "#{ends[0].strip}\n"
@license_text.each do |text|
target_file.print "#{text.rstrip}\n"
end
target_file.print "#{ends[1].strip}\n"
end
# This file either doesn't match any of the file-matching strings, or
# matches an exception
#
def copy_file_without_license(source_path, target_path)
FileUtils.cp(source_path, target_path)
end
# ------------------------------------------------------------------------------------
public
# ------------------------------------------------------------------------------------
# Setup and get ready to process.
#
# * properties is a map of keys to values, probably parsed from a properties file.
# * apply is a boolean.
# If false, just scan the source directory for problems.
# If true, copy from the source directory to the target, applying the license.
#
def initialize(properties, apply)
@scan_only = !apply
@source_dir = properties['source_dir']
@target_dir = properties['target_dir']
@file_match_list = properties['file_matchers']
@license_file = properties['license_file']
@known_exceptions_file = properties['known_exceptions']
@full_report_string = properties['full_report']
sanity_checks_on_parameters()
@full_report = @full_report_string === 'true' || @full_report_string === 'yes'
@file_matchers = @file_match_list.strip.split(/,\s*/)
@license_text = prepare_license_text(@license_file)
@known_exceptions = prepare_exception_globs(@known_exceptions_file, @source_dir)
@stats = LicenserStats.new(@source_dir, @file_matchers, @full_report)
end
# Start the recursive scanning (and copying).
def process()
scan_dir(@source_dir, @target_dir)
end
# Report the summary statistics
def report()
verb = @scan_only ? "scanned" : "copied"
puts "Licenser: run completed at #{DateTime.now.strftime("%H:%M:%S on %b %d, %Y")}"
puts " #{verb} #{@stats.file_count} files in #{@stats.dir_count} directories."
puts
puts 'Substitutions'
@stats.substitutions.sort.each do |line|
printf("%5d %s\n", line[1], line[0])
end
puts
puts 'Known non-licensed files'
@stats.known_exceptions.sort.each do |line|
printf("%5d %s\n", line[1], line[0])
end
puts
puts 'Missing tags'
@stats.missing_tags.sort.each do |line|
printf("%5d %s\n", line[1], line[0])
end
puts
puts 'parameters:'
puts " source_dir = #{@source_dir}"
puts " target_dir = #{@target_dir}"
puts " file_matchers = #{@file_matchers.join(', ')}"
puts " license_file = #{@license_file}"
puts " known_exceptions_file = #{@known_exceptions_file}"
puts " scan_only = #{@scan_only}"
puts " full_report = #{@full_report}"
end
# Were we successful or not?
def success?
return @stats.missing_tags.empty?
end
end
#
#
# ------------------------------------------------------------------------------------
# Standalone calling.
#
# Do this if this program was called from the command line. That is, if the command
# expands to the path of this file.
# ------------------------------------------------------------------------------------
#
if File.expand_path($0) == File.expand_path(__FILE__)
if ARGV.length == 0
raise("No arguments - usage is: ruby licenser.rb <properties_file> [apply_to_target]")
end
if !File.file?(ARGV[0])
raise "File does not exist: '#{ARGV[0]}'."
end
properties = PropertyFileReader.read(ARGV[0])
if ARGV.length > 1
if ARGV[1] == "apply_to_target"
apply = true
else
apply = false
end
end
l = Licenser.new(properties, apply)
l.process
l.report
if l.success?
puts "Licenser was successful."
exit 0
else
puts "Licenser found problems."
exit 1
end
end