litvinovg/vivo
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

Maven migration (first draft)

Browse source
This commit is contained in:
Graham Triggs 2015-11-19 23:47:41 +00:00
parent da79ac3e1d
commit fee48b0b50
1711 changed files with 662 additions and 0 deletions
Download patch file Download diff file Expand all files Collapse all files

2
legacy/utilities/languageSupport/i18nChecker/check Executable file
View file

@ -0,0 +1,2 @@
ruby i18nChecker.rb '/Library/Tomcat/webapps/vivo/i18n/*.properties' complete
ruby i18nChecker.rb '/Library/Tomcat/webapps/vivo/themes/wilma/i18n/*.properties' complete

149
legacy/utilities/languageSupport/i18nChecker/i18nChecker.rb Normal file
View file

@ -0,0 +1,149 @@
=begin
--------------------------------------------------------------------------------
A utility routine that scans I18n-related properties files and Freemarker
templates for obvious issues.
Properties files:
Warn if a specialized file has no default version.
Warn about duplicate keys, keys with empty values.
Warn about file reference values with no corresponding file
Warn about keys that do not appear in the default version.
If the "complete" flag is set,
Warn if the default version is not found.
Warn about missing keys, compared to the default version.
Freemarker templates:
Warn about visible text that contains other than blank space or Freemarker expressions.
Visible text is:
Anything that is not inside a tag and not between <script> tags
title="" attributes on any tags
alert="" attributes on <img> tags
alt="" attributes on <img> tags
value="" attributes on <input> tags with submit attributes
On the command line, provide a file "glob" (don't allow the shell to interpret it)
and optional "complete" or "summary" flags. E.g.:
i18nChecker.rb '../../themes/wilma/**/*' complete
--------------------------------------------------------------------------------
=end
$: << File.dirname(File.expand_path(__FILE__))
require 'properties_file_checker'
require 'template_file_checker'
require 'template_set_checker'
class I18nChecker
# ------------------------------------------------------------------------------------
private
# ------------------------------------------------------------------------------------
#
# Parse the arguments and complain if they don't make sense.
#
def sanity_check_arguments(args)
if ARGV.length == 0
raise("No arguments - usage is: ruby i18nChecker.rb <file_spec_glob> [complete] [summary]")
end
file_spec = args[0]
complete = false;
summary = false;
args[1..-1].each do |arg|
if "complete" == arg.downcase()
complete = true
elsif "summary" == arg.downcase()
summary = true
else
raise("'#{arg}' is an invalid argument")
end
end
puts "file_spec = '#{file_spec}', complete = #{complete}, summary = #{summary}"
return file_spec, complete, summary
end
#
# Go through the specified files and pick out the *.properties and *.ftl files.
#
def get_file_paths(file_spec)
properties = []
templates = []
Dir.glob(file_spec) do |path|
properties << path if File.extname(path) == '.properties'
templates << path if File.extname(path) == '.ftl'
end
@total_files = properties.size + templates.size
puts "Found #{properties.size} property files, #{templates.size} templates."
return properties, templates
end
def process_properties_files(paths, complete, summary)
checkers = []
paths.each() do |path|
checkers << PropertiesFileChecker.new(path)
end
checkers.each() do |child|
checkers.each() do |root|
child.try_to_set_root(root)
end
end
checkers.each() do |checker|
checker.report(complete, summary)
@total_warnings += checker.warnings.size
end
end
def process_template_files(paths, summary)
ts = TemplateSetChecker.new(paths)
ts.report(summary)
paths = ts.remaining_paths
@total_warnings += ts.warnings_count
paths.each() do |path|
tf = TemplateFileChecker.new(path)
tf.report(summary)
@total_warnings += tf.warnings.size
end
end
# ------------------------------------------------------------------------------------
public
# ------------------------------------------------------------------------------------
def initialize(args)
@total_files = 0
@total_warnings = 0
file_spec, complete, summary = sanity_check_arguments(args)
properties, templates = get_file_paths(file_spec)
process_properties_files(properties, complete, summary)
process_template_files(templates, summary)
end
def summarize()
puts "Found #{@total_warnings} warnings in #{@total_files} files."
end
end
#
#
# ------------------------------------------------------------------------------------
# Standalone calling.
#
# Do this if this program was called from the command line. That is, if the command
# expands to the path of this file.
# ------------------------------------------------------------------------------------
#
if File.expand_path($0) == File.expand_path(__FILE__)
checker = I18nChecker.new(ARGV)
checker.summarize()
end

220
legacy/utilities/languageSupport/i18nChecker/properties_file_checker.rb Normal file
View file

@ -0,0 +1,220 @@
=begin
--------------------------------------------------------------------------------
Read and interpret a properties file. Accept a default version, if appropriate.
Warn if a specialized file has no default version.
Warn about duplicate keys, keys with empty values.
Warn about file reference values with no corresponding file
Warn about keys that do not appear in the default version.
If the "complete" flag is set,
Warn if the default version is not found.
Warn about missing keys, compared to the default version.
--------------------------------------------------------------------------------
=end
class Warning
attr_reader :line
attr_reader :message
def initialize(line, message)
@line = line
@message = message
end
end
class Property
attr_reader :line
attr_reader :key
attr_reader :value
def initialize(line, key, value)
@line = line
@key = key
@value = value
end
end
class PropertiesFileChecker
# ------------------------------------------------------------------------------------
private
# ------------------------------------------------------------------------------------
#
# What path would represent the default version of this property file?
#
def figure_rootpath()
name = File.basename(path)
dirname = File.dirname(path)
extname = File.extname(path)
raise("Invalid property file name: '#{path}': too many underscores.") if name.count("_") > 2
first_underscore = name.index('_')
if first_underscore
@rootpath = File.join(dirname, name[0, first_underscore] + extname)
else
@rootpath = path
end
end
#
# Is this a default version or a locale-specific version?
#
def is_root?()
@rootpath == @path
end
def check_for_faux_continuations(lines)
ln = 0
lines.map do |line|
ln += 1
if /(\\) +$/.match(line)
@warnings << Warning.new(ln, "On a continuation line, the \\ must not be followed by spaces.")
$` + $1
else
line
end
end
end
def join_continuation_lines(lines)
(lines.size()-1).downto(0) do |i|
if /(.*)\\$/.match(lines[i])
lines[i] = $1 + lines[i+1].lstrip()
lines[i+1] = ''
end
end
return lines
end
def read_properties(lines)
ln = 0
lines.each do |line|
ln += 1
line.strip!
# ignore blank lines, and lines starting with '#' or '!'.
next if line.length == 0 || line[0] == ?# || line[0] == ?!
if line =~ /(.*?)\s*[=:]\s*(.*)/
# key and value are separated by '=' or ':' and optional whitespace.
key = $1.strip
value = $2
else
# No '=' or ':' means that the value is empty.
key = line;
value = ''
end
if dupe = @properties[key]
@warnings << Warning.new(ln, "Key '#{key}' is duplicated on line #{dupe.line}")
else
@properties[key] = Property.new(ln, key, value)
end
end
end
def check_for_root_file()
@warnings << Warning.new(0, "Found no root file '#{File.basename(@rootpath)}'") if !is_root?() && !@root
end
def scan_for_empty_values()
@properties.values.each do |p|
@warnings << Warning.new(p.line, "Value for '#{p.key}' is empty.") if p.value.empty?
end
end
def scan_for_invalid_file_references()
@properties.values.each do |p|
if /@@file\s+(.*)/.match(p.value)
file_reference = $1.strip
dir = File.dirname(@path)
path = File.join(dir, file_reference)
unless File.file?(path)
@warnings << Warning.new(p.line, "Invalid file reference '#{p.value}': file not found")
end
end
end
end
def scan_for_properties_not_present_in_root()
root_name = File.basename(@root.path)
extra_keys = @properties.keys - @root.properties.keys
extra_keys.each do |key|
p = @properties[key]
@warnings << Warning.new(p.line, "Property '#{key}' is not present in root file '#{root_name}'")
end
end
def scan_for_properties_not_present_in_derived
root_name = File.basename(@root.path)
next_line = @properties.values.max {|a, b| a.line <=> b.line}.line + 1
missing_keys = @root.properties.keys - @properties.keys
missing_keys.sort.each do |key|
@warnings << Warning.new(next_line, "No value to override '#{key}' in the root file '#{root_name}'")
end
end
# ------------------------------------------------------------------------------------
public
# ------------------------------------------------------------------------------------
def initialize(path)
@path = path
@root = nil
@rootpath = nil
@warnings = []
@properties = {}
figure_rootpath()
lines = IO.readlines(@path)
lines = check_for_faux_continuations(lines)
lines = join_continuation_lines(lines)
read_properties(lines)
end
def try_to_set_root(root)
if !is_root?
if root.path == @rootpath
@root = root
end
end
end
def path()
@path
end
def properties()
@properties
end
def warnings()
@warnings
end
#
# Analyze the properties, and say what we found.
#
def report(complete, summary)
check_for_root_file() if complete
scan_for_empty_values()
scan_for_invalid_file_references()
scan_for_properties_not_present_in_root() if @root
scan_for_properties_not_present_in_derived() if complete && @root
puts " Properties file '#{@path}', #{@properties.size()} properties"
if !@warnings.empty?
if summary
puts " #{@warnings.size} warnings."
else
@warnings.sort! {|a, b| a.line <=> b.line}
@warnings.each do |w|
puts " line #{w.line}: #{w.message}"
end
end
end
end
end

176
legacy/utilities/languageSupport/i18nChecker/template_file_checker.rb Normal file
View file

@ -0,0 +1,176 @@
=begin
--------------------------------------------------------------------------------
Read a freemarker template and looks for likely i18n problems.
Warn about visible text that contains other than blank space or Freemarker expressions.
Visible text is:
Anything that is not inside a tag and not between <script> or <style> tags
title="" attributes on any tags
alert="" attributes on <img> tags
alt="" attributes on <img> tags
value="" attributes on <input> tags with submit attributes
a Freemarker express is enclosed in ${}
--------------------------------------------------------------------------------
=end
class Warning
attr_reader :line
attr_reader :message
def initialize(line, message)
@line = line
@message = message
end
end
class TemplateFileChecker
# ------------------------------------------------------------------------------------
private
# ------------------------------------------------------------------------------------
def blanks(length)
' '.ljust(length, ' ')
end
def replace_comments(raw)
raw.gsub(/<!--.*?-->/m) {|s| blanks(s.size)}
end
def replace_script_tags(raw)
raw.gsub(/<script.*?>.*?<\/script>/m) {|s| blanks(s.size)}
end
def replace_style_tags(raw)
raw.gsub(/<style.*?>.*?<\/style>/m) {|s| blanks(s.size)}
end
def replace_freemarker_comments(raw)
raw.gsub(/<#--.*?-->/m) {|s| blanks(s.size)}
end
def replace_freemarker_tags(raw)
raw.gsub(/<#.*?>/m) {|s| blanks(s.size)}
end
def replace_freemarker_expressions(raw)
dirty = raw
while /\$\{[^\{\}]*\}/m.match(dirty)
dirty = $` + blanks($&.size) + $'
end
dirty
end
def remove_entities(raw)
raw.gsub(/&[a-zA-Z]+;/, '')
end
def launder(raw)
replace_script_tags(
replace_style_tags(
replace_freemarker_expressions(
replace_freemarker_tags(
replace_comments(
replace_freemarker_comments(raw))))))
end
def load(path)
IO.open(File.new(path).fileno) do |f|
@contents = f.read()
@clean_contents = launder(@contents)
pos = 0
while found = @contents.index(/\n/, pos)
pos = found + 1
@line_offsets << pos
end
end
end
def line_number(offset)
return @line_offsets.find_index() {|o| o > offset} || @line_offsets.size
end
def text_for_display(raw)
clean = raw.gsub(/[\n\r]/m, ' ').strip
if clean.size < 50
clean
else
clean[0..50] + "..."
end
end
def scan(regexp, group_index, message)
@clean_contents.gsub(regexp) do |s|
offset = $~.begin(group_index)
value = $~[group_index]
if contains_words?(value)
@warnings << Warning.new(line_number(offset), "#{message}: '#{text_for_display(value)}'")
end
s
end
end
def contains_words?(raw)
remove_entities(raw).count('a-zA-Z') > 0
end
def scan_for_words_outside_of_tags()
scan(/>\s*([^><]+)</m, 1, "Words found outside of tags")
end
def scan_for_title_attributes()
scan(/<[^>]*title=(["'])\s*([^>]*?)\s*\1.*?>/mi, 2, "Words found in title attribute of an HTML tag")
end
def scan_for_alert_attributes()
scan(/<img\b[^>]*alert=(["'])\s*([^>]*?)\s*\1.*?>/mi, 2, "Words found in alert attribute of <img> tag")
end
def scan_for_alt_attributes()
scan(/<img\b[^>]*alt=(["'])\s*([^>]*?)\s*\1.*?>/mi, 2, "Words found in alt attribute of <img> tag")
end
def scan_for_value_attributes_on_submit_tags()
scan(/<input\b[^>]*type=["']submit["'][^>]*value=(["'])\s*([^'">]*?)\s*\1.*?>/mi, 2, "Words found in value attribute of <input type='submit'> tag")
end
# ------------------------------------------------------------------------------------
public
# ------------------------------------------------------------------------------------
def initialize(path)
@path = path
@contents = ''
@clean_contents = ''
@line_offsets = [0]
@warnings = []
load(path)
scan_for_words_outside_of_tags()
scan_for_title_attributes()
scan_for_alert_attributes()
scan_for_alt_attributes()
scan_for_value_attributes_on_submit_tags()
end
def report(summary)
puts " Template file '#{@path}'"
if !@warnings.empty?
if summary
puts " #{@warnings.size} warnings."
else
@warnings.sort! {|a, b| a.line <=> b.line}
@warnings.each do |w|
puts " line #{w.line}: #{w.message}"
end
end
end
$stdout.flush
end
def warnings()
@warnings
end
end

174
legacy/utilities/languageSupport/i18nChecker/template_set_checker.rb Normal file
View file

@ -0,0 +1,174 @@
=begin
--------------------------------------------------------------------------------
Look at a set of template paths, and remove any paths that represent
language-specific templates. Warn if any of those templates are using the i18n()
Freemarker method.
Note: any template path that ends in _xx.ftl or _xx_YY.ftl is assumed to be
language-specific. As a heuristic, we will also assume that any template with
the same path, but without the language-specifier, is also language-specific,
representing the default version of the template.
--------------------------------------------------------------------------------
=end
class Warning
attr_reader :line
attr_reader :message
def initialize(line, message)
@line = line
@message = message
end
end
class TemplateSet
attr_reader :root
attr_reader :paths
attr_reader :warnings
def initialize(root)
@root = root
@paths = []
@warnings = []
end
def add_path(path)
@paths << path
end
def add_warning(warning)
@warnings << warning
end
end
class TemplateSetChecker
# ------------------------------------------------------------------------------------
private
# ------------------------------------------------------------------------------------
def find_template_sets()
find_language_specific_templates()
find_default_language_templates()
remove_from_remaining_paths()
end
def find_language_specific_templates()
@paths.each() do |path|
if is_language_specific(path)
add_to_template_sets(path)
end
end
end
def is_language_specific(path)
/_[a-z]{2}(_[A-Z]{2})?\.ftl$/ =~ path
end
def add_to_template_sets(path)
root = root_path(path)
set = @template_sets[root] || TemplateSet.new(root)
set.add_path(path)
@template_sets[root] = set
end
def root_path(path)
if /(.*?)(_[a-z]{2}(_[A-Z]{2})?)?\.ftl$/ =~ path
$1
else
path
end
end
def find_default_language_templates()
@paths.each() do |path|
root = root_path(path)
set = @template_sets[root]
if set
set.add_path(path) unless set.paths.include?(path)
end
end
end
def remove_from_remaining_paths()
@remaining_paths = Array.new(@paths)
@template_sets.each_value do |set|
set.paths.each do |path|
@remaining_paths.delete(path)
end
end
end
def scan_set(set)
set.paths.each do |path|
load(path)
@contents.gsub(/i18n/) do |s|
offset = $~.begin(0)
set.add_warning(Warning.new(line_number(offset), "in #{path}: Call to i18n() in a language-specific template."))
s
end
end
end
def load(path)
@line_offsets = []
IO.open(File.new(path).fileno) do |f|
@contents = f.read()
pos = 0
while found = @contents.index(/\n/, pos)
pos = found + 1
@line_offsets << pos
end
end
end
def line_number(offset)
return @line_offsets.find_index() {|o| o > offset} || @line_offsets.size
end
# ------------------------------------------------------------------------------------
public
# ------------------------------------------------------------------------------------
def initialize(paths)
@paths = paths
@template_sets = {}
@remaining_paths = []
find_template_sets()
@template_sets.each_value do |set|
scan_set(set)
end
end
def report(summary)
@template_sets.each_value do |set|
puts" Template file set '#{set.paths.join("'\n '")}'"
if !set.warnings.empty?
if summary
puts " #{set.warnings.size} warnings."
else
set.warnings.sort! {|a, b| a.line <=> b.line}
set.warnings.each do |w|
puts " line #{w.line}: #{w.message}"
end
end
end
end
$stdout.flush
end
def warnings_count()
count = 0
@template_sets.each_value do |set|
count += set.warnings.size
end
count
end
def remaining_paths()
@remaining_paths
end
end

47
legacy/utilities/languageSupport/translateDisplayNames/display_name_common.rb Executable file
View file

@ -0,0 +1,47 @@
#!/usr/bin/ruby
=begin
--------------------------------------------------------------------------------
--------------------------------------------------------------------------------
=end
require 'rubygems'
require 'rdf'
require 'rdf/rdfxml'
require 'rdf/ntriples'
require 'rdf/n3'
include RDF
DISPLAY_NAME_URI = RDF::URI.new("http://vitro.mannlib.cornell.edu/ns/vitro/ApplicationConfiguration#displayName")
class UsageError < StandardError; end
class FilterError < StandardError; end
class DisplayNameCommon
# ------------------------------------------------------------------------------------
private
# ------------------------------------------------------------------------------------
# ------------------------------------------------------------------------------------
public
# ------------------------------------------------------------------------------------
def self.load_filter(filter_file)
return lambda{|s| true} unless filter_file
return eval(IO.read(filter_file))
rescue
raise FilterError.new($!.message)
end
def initialize(rdf_file, &filter)
@filter = filter.nil? ? lambda{true} : filter
@graph = Graph.load(rdf_file)
end
def process(query, &filter)
solutions = query.execute(@graph)
solutions.filter!(&filter)
solutions.order(:prop)
end
end

118
legacy/utilities/languageSupport/translateDisplayNames/display_name_inserter.rb Normal file
View file

@ -0,0 +1,118 @@
#!/usr/bin/ruby
=begin
--------------------------------------------------------------------------------
A utility that reads an RDF file, builds a model, sorts all of the URIs that have
display_names, and associates those URIs with the display_names in a supplied text file, one line
per display_name.
These display_names are assigned the language specified on the command line, and the
resulting RDF statements are sent to standard output as N3.
On the command line provide the path to the RDF file, the path to the display_names file,
and the desired language/locale. E.g.:
display_name_inserter.rb ../../vivo-core-1.5-annotations.rdf display_names.file es_ES
--------------------------------------------------------------------------------
=end
$: << File.dirname(File.expand_path(__FILE__))
require 'rubygems'
require 'rdf'
require 'display_name_common'
include RDF
class DisplayNameInserter
# ------------------------------------------------------------------------------------
private
# ------------------------------------------------------------------------------------
#
# Parse the arguments and complain if they don't make sense.
#
def sanity_check_arguments(args)
raise UsageError, "usage is: display_name_inserter.rb <rdf_file> <display_names_input_file> <locale> [filter_file] <n3_output_file> [ok]" unless (3..5).include?(args.length)
if args[-1].downcase == 'ok'
ok = true
args.pop
else
ok = false
end
n3_output_file = args.pop
raise UsageError, "File '#{n3_output_file}' already exists. specify 'ok' to overwrite it." if File.exist?(n3_output_file) && !ok
rdf_file = args[0]
raise UsageError, "File '#{rdf_file}' does not exist." unless File.exist?(rdf_file)
display_names_input_file = args[1]
raise UsageError, "File '#{display_names_input_file}' does not exist." unless File.exist?(display_names_input_file)
locale = args[2]
raise UsageError, "Locale should be like 'ab' or 'ab-CD'." unless /^[a-z]{2}(-[A-Z]{2})?$/ =~ locale
filter_file = args[3]
raise UsageError, "File '#{filter_file}' does not exist." if filter_file && !File.exist?(filter_file)
filter = DisplayNameCommon.load_filter(filter_file)
return rdf_file, display_names_input_file, locale, filter, n3_output_file
end
# ------------------------------------------------------------------------------------
public
# ------------------------------------------------------------------------------------
def initialize(args)
@rdf_file, @display_names_input_file, @locale, @filter, @n3_output_file = sanity_check_arguments(args)
rescue UsageError => e
puts "\n----------------\nUsage error\n----------------\n\n#{e}\n\n----------------\n\n"
exit
rescue FilterError => e
puts "\n----------------\nFilter file is invalid\n----------------\n\n#{e}\n\n----------------\n\n"
exit
end
def process()
query = Query.new({
:prop => {
DISPLAY_NAME_URI => :display_name,
}
})
solutions = DisplayNameCommon.new(@rdf_file).process(query, &@filter)
display_names = IO.readlines(@display_names_input_file)
raise "Number of display_names (#{display_names.length}) doesn't match number of URIs (#{solutions.length})" unless display_names.length == solutions.length
graph = Graph.new
solutions.zip(display_names).each do |data|
s = data[0].prop
p = DISPLAY_NAME_URI
o = Literal.new(data[1].chomp, :language => @locale)
graph << Statement.new(s, p, o)
end
File.open(@n3_output_file, 'w') do |f|
f.puts graph.dump(:ntriples)
end
end
end
#
#
# ------------------------------------------------------------------------------------
# Standalone calling.
#
# Do this if this program was called from the command line. That is, if the command
# expands to the path of this file.
# ------------------------------------------------------------------------------------
#
if File.expand_path($0) == File.expand_path(__FILE__)
inserter = DisplayNameInserter.new(ARGV)
inserter.process()
end

101
legacy/utilities/languageSupport/translateDisplayNames/display_name_stripper.rb Executable file
View file

@ -0,0 +1,101 @@
#!/usr/bin/ruby
=begin
--------------------------------------------------------------------------------
A utility that reads an RDF file, builds a model, sorts all of the URIs that have
display_names, and produces a file of those display_names, one per line. The idea is that this
file could be translated, and the result could be put into RDF by display_name_inserter.rb
This required the RDF.rb gem: sudo gem install rdf
On the command line provide the path to the RDF file. E.g.:
display_name_stripper.rb '../../PropertyConfig.n3'
--------------------------------------------------------------------------------
=end
$: << File.dirname(File.expand_path(__FILE__))
require 'rubygems'
require 'rdf'
require 'display_name_common'
require 'rdf/n3'
include RDF
class DisplayNameStripper
# ------------------------------------------------------------------------------------
private
# ------------------------------------------------------------------------------------
#
# Parse the arguments and complain if they don't make sense.
#
def sanity_check_arguments(args)
raise UsageError, "usage is: display_name_stripper.rb <rdf_file> [filter_file] <display_names_output_file> [ok]" unless (2..3).include?(args.length)
if args[-1].downcase == 'ok'
ok = true
args.pop
else
ok = false
end
output_file = args.pop
raise UsageError, "File '#{output_file}' already exists. specify 'ok' to overwrite it." if File.exist?(output_file) && !ok
rdf_file = args[0]
raise UsageError, "File '#{rdf_file}' does not exist." unless File.exist?(rdf_file)
filter_file = args[1]
raise UsageError, "File '#{filter_file}' does not exist." if filter_file && !File.exist?(filter_file)
filter = DisplayNameCommon.load_filter(filter_file)
return rdf_file, filter, output_file
end
# ------------------------------------------------------------------------------------
public
# ------------------------------------------------------------------------------------
def initialize(args)
@rdf_file, @filter, @display_names_output_file = sanity_check_arguments(args)
rescue UsageError => e
puts "\n----------------\nUsage error\n----------------\n\n#{e}\n\n----------------\n\n"
exit
rescue FilterError => e
puts "\n----------------\nFilter file is invalid\n----------------\n\n#{e}\n\n----------------\n\n"
exit
end
def process()
query = Query.new({
:prop => {
DISPLAY_NAME_URI => :display_name,
}
})
solutions = DisplayNameCommon.new(@rdf_file).process(query, &@filter)
File.open(@display_names_output_file, 'w') do |f|
solutions.each do |s|
f.puts s.display_name
end
end
end
end
#
#
# ------------------------------------------------------------------------------------
# Standalone calling.
#
# Do this if this program was called from the command line. That is, if the command
# expands to the path of this file.
# ------------------------------------------------------------------------------------
#
if File.expand_path($0) == File.expand_path(__FILE__)
stripper = DisplayNameStripper.new(ARGV)
stripper.process()
end

44
legacy/utilities/languageSupport/translateLabelsInOntology/label_common.rb Executable file
View file

@ -0,0 +1,44 @@
#!/usr/bin/ruby
=begin
--------------------------------------------------------------------------------
--------------------------------------------------------------------------------
=end
require 'rubygems'
require 'rdf'
require 'rdf/rdfxml'
require 'rdf/ntriples'
require 'rdf/n3'
include RDF
class UsageError < StandardError; end
class FilterError < StandardError; end
class LabelCommon
# ------------------------------------------------------------------------------------
private
# ------------------------------------------------------------------------------------
# ------------------------------------------------------------------------------------
public
# ------------------------------------------------------------------------------------
def self.load_filter(filter_file)
return lambda{|s| true} unless filter_file
return eval(IO.read(filter_file))
rescue
raise FilterError.new($!.message)
end
def initialize(rdf_file, &filter)
@filter = filter.nil? ? lambda{true} : filter
@graph = Graph.load(rdf_file)
end
def process(query, &filter)
solutions = query.execute(@graph)
solutions.filter!(&filter)
solutions.order(:prop)
end
end

118
legacy/utilities/languageSupport/translateLabelsInOntology/label_inserter.rb Executable file
View file

@ -0,0 +1,118 @@
#!/usr/bin/ruby
=begin
--------------------------------------------------------------------------------
A utility that reads an RDF file, builds a model, sorts all of the URIs that have
labels, and associates those URIs with the labels in a supplied text file, one line
per label.
These labels are assigned the language specified on the command line, and the
resulting RDF statements are sent to standard output as N3.
On the command line provide the path to the RDF file, the path to the labels file,
and the desired language/locale. E.g.:
label_inserter.rb ../../vivo-core-1.5-annotations.rdf labels.file es_ES
--------------------------------------------------------------------------------
=end
$: << File.dirname(File.expand_path(__FILE__))
require 'rubygems'
require 'rdf'
require 'label_common'
include RDF
class LabelInserter
# ------------------------------------------------------------------------------------
private
# ------------------------------------------------------------------------------------
#
# Parse the arguments and complain if they don't make sense.
#
def sanity_check_arguments(args)
raise UsageError, "usage is: label_inserter.rb <rdf_file> <labels_input_file> <locale> [filter_file] <n3_output_file> [ok]" unless (3..5).include?(args.length)
if args[-1].downcase == 'ok'
ok = true
args.pop
else
ok = false
end
n3_output_file = args.pop
raise UsageError, "File '#{n3_output_file}' already exists. specify 'ok' to overwrite it." if File.exist?(n3_output_file) && !ok
rdf_file = args[0]
raise UsageError, "File '#{rdf_file}' does not exist." unless File.exist?(rdf_file)
labels_input_file = args[1]
raise UsageError, "File '#{labels_input_file}' does not exist." unless File.exist?(labels_input_file)
locale = args[2]
raise UsageError, "Locale should be like 'ab' or 'ab-CD'." unless /^[a-z]{2}(-[A-Z]{2})?$/ =~ locale
filter_file = args[3]
raise UsageError, "File '#{filter_file}' does not exist." if filter_file && !File.exist?(filter_file)
filter = LabelCommon.load_filter(filter_file)
return rdf_file, labels_input_file, locale, filter, n3_output_file
end
# ------------------------------------------------------------------------------------
public
# ------------------------------------------------------------------------------------
def initialize(args)
@rdf_file, @labels_input_file, @locale, @filter, @n3_output_file = sanity_check_arguments(args)
rescue UsageError => e
puts "\n----------------\nUsage error\n----------------\n\n#{e}\n\n----------------\n\n"
exit
rescue FilterError => e
puts "\n----------------\nFilter file is invalid\n----------------\n\n#{e}\n\n----------------\n\n"
exit
end
def process()
query = Query.new({
:prop => {
RDFS.label => :label,
}
})
solutions = LabelCommon.new(@rdf_file).process(query, &@filter)
labels = IO.readlines(@labels_input_file)
raise "Number of labels (#{labels.length}) doesn't match number of URIs (#{solutions.length})" unless labels.length == solutions.length
graph = Graph.new
solutions.zip(labels).each do |data|
s = data[0].prop
p = RDFS.label
o = Literal.new(data[1].chomp, :language => @locale)
graph << Statement.new(s, p, o)
end
File.open(@n3_output_file, 'w') do |f|
f.puts graph.dump(:ntriples)
end
end
end
#
#
# ------------------------------------------------------------------------------------
# Standalone calling.
#
# Do this if this program was called from the command line. That is, if the command
# expands to the path of this file.
# ------------------------------------------------------------------------------------
#
if File.expand_path($0) == File.expand_path(__FILE__)
inserter = LabelInserter.new(ARGV)
inserter.process()
end

101
legacy/utilities/languageSupport/translateLabelsInOntology/label_stripper.rb Executable file
View file

@ -0,0 +1,101 @@
#!/usr/bin/ruby
=begin
--------------------------------------------------------------------------------
A utility that reads an RDF file, builds a model, sorts all of the URIs that have
labels, and produces a file of those labels, one per line. The idea is that this
file could be translated, and the result could be put into RDF by label_inserter.rb
This required the RDF.rb gem: sudo gem install rdf
On the command line provide the path to the RDF file. E.g.:
label_stripper.rb '../../vivo-core-1.5-annotations.rdf'
--------------------------------------------------------------------------------
=end
$: << File.dirname(File.expand_path(__FILE__))
require 'rubygems'
require 'rdf'
require 'label_common'
require 'rdf/n3'
include RDF
class LabelStripper
# ------------------------------------------------------------------------------------
private
# ------------------------------------------------------------------------------------
#
# Parse the arguments and complain if they don't make sense.
#
def sanity_check_arguments(args)
raise UsageError, "usage is: label_stripper.rb <rdf_file> [filter_file] <labels_output_file> [ok]" unless (2..3).include?(args.length)
if args[-1].downcase == 'ok'
ok = true
args.pop
else
ok = false
end
output_file = args.pop
raise UsageError, "File '#{output_file}' already exists. specify 'ok' to overwrite it." if File.exist?(output_file) && !ok
rdf_file = args[0]
raise UsageError, "File '#{rdf_file}' does not exist." unless File.exist?(rdf_file)
filter_file = args[1]
raise UsageError, "File '#{filter_file}' does not exist." if filter_file && !File.exist?(filter_file)
filter = LabelCommon.load_filter(filter_file)
return rdf_file, filter, output_file
end
# ------------------------------------------------------------------------------------
public
# ------------------------------------------------------------------------------------
def initialize(args)
@rdf_file, @filter, @labels_output_file = sanity_check_arguments(args)
rescue UsageError => e
puts "\n----------------\nUsage error\n----------------\n\n#{e}\n\n----------------\n\n"
exit
rescue FilterError => e
puts "\n----------------\nFilter file is invalid\n----------------\n\n#{e}\n\n----------------\n\n"
exit
end
def process()
query = Query.new({
:prop => {
RDFS.label => :label,
}
})
solutions = LabelCommon.new(@rdf_file).process(query, &@filter)
File.open(@labels_output_file, 'w') do |f|
solutions.each do |s|
f.puts s.label
end
end
end
end
#
#
# ------------------------------------------------------------------------------------
# Standalone calling.
#
# Do this if this program was called from the command line. That is, if the command
# expands to the path of this file.
# ------------------------------------------------------------------------------------
#
if File.expand_path($0) == File.expand_path(__FILE__)
stripper = LabelStripper.new(ARGV)
stripper.process()
end

12
legacy/utilities/languageSupport/translateLabelsInOntology/vivo_core_filter Normal file
View file

@ -0,0 +1,12 @@
lambda {|s|
(
s.prop.start_with?("http://vivoweb.org/ontology/core#") ||
s.prop.start_with?("http://vitro.mannlib.cornell.edu/ns/") ||
s.prop.start_with?("http://purl.org/ontology/bibo/") ||
s.prop.start_with?("http://xmlns.com/foaf/0.1/") ||
s.prop.start_with?("http://purl.org/dc/terms/") ||
s.prop.start_with?("http://purl.org/dc/elements/1.1/") ||
s.prop.start_with?("http://purl.org/NET/c4dm/event.owl#") ) &&
!s.label.to_s.strip.empty?
}

80
legacy/utilities/languageSupport/translateValuesInPropertyFile/property_common.rb Normal file
View file

@ -0,0 +1,80 @@
#!/usr/bin/ruby
=begin
--------------------------------------------------------------------------------
Some common routines used both by property_stripper and property_inserter
--------------------------------------------------------------------------------
=end
class Warning
attr_reader :line
attr_reader :message
def initialize(line, message)
@line = line
@message = message
end
end
class Property
attr_reader :line
attr_reader :key
attr_accessor :value
def initialize(line, key, value)
@line = line
@key = key
@value = value
end
end
class PropertiesFile
attr_reader :properties
attr_reader :warnings
def join_continuation_lines(lines)
(lines.size()-1).downto(0) do |i|
if /(.*)\\$/.match(lines[i])
lines[i] = $1 + lines[i+1].lstrip()
lines[i+1] = ''
end
end
return lines
end
def read_properties(lines)
ln = 0
lines.each do |line|
ln += 1
line.strip!
# ignore blank lines, and lines starting with '#' or '!'.
next if line.length == 0 || line[0] == ?# || line[0] == ?!
if line =~ /(.*?)\s*[=:]\s*(.*)/
# key and value are separated by '=' or ':' and optional whitespace.
key = $1.strip
value = $2
else
# No '=' or ':' means that the value is empty.
key = line;
value = ''
end
if dupe = @properties[key]
@warnings << Warning.new(ln, "Key '#{key}' is duplicated on line #{dupe.line}")
else
@properties[key] = Property.new(ln, key, value)
end
end
end
def initialize(path)
@properties = {}
@warnings = []
lines = IO.readlines(path)
lines = join_continuation_lines(lines)
read_properties(lines)
end
end

165
legacy/utilities/languageSupport/translateValuesInPropertyFile/property_inserter.rb Executable file
View file

@ -0,0 +1,165 @@
#!/usr/bin/ruby
=begin
--------------------------------------------------------------------------------
Builds a property file, using an existing property file as a template, but
getting the property values from a text file of translated text
(see property_stripper.rb) and optionally a partial file of translated properties.
So, if you have a template file of English-language properties (e.g. all.properties),
this will read the file into a properties structure. The text file of translated
values is presumed to have one value per line, associated with the alphabetized
list of keys from the template file. The translated values will replace the orignal
values, with the exception that any value that starts with @@file will not be
replaced.
If a partially translated file is provided, it will be read and used to replace
any translated values from the text file, which are assumed to be weaker. Note
that this is true of @@file values as well, which are presumed to be corrected
for the language.
Any @@file values that are not overridden by the partial translation will result in
a warning to stderr.
Finally, the template file is processed again, replacing the existing values with
the translated values, but keeping the same comment and spacing structure.
On the command line provide the path to the tempate file, the text file, and
optionally the partial translation. E.g.:
property_inserter.rb '../../all.properties' translated.txt '../../all_es.properties' 'all_es.properties'
--------------------------------------------------------------------------------
=end
$: << File.dirname(File.expand_path(__FILE__))
require "property_common"
class PropertyInserter
# ------------------------------------------------------------------------------------
private
# ------------------------------------------------------------------------------------
#
# Parse the arguments and complain if they don't make sense.
#
def sanity_check_arguments(args)
raise "usage is: property_inserter.rb <template_file> <translated_values_file> [partial_translation] <output_file> [ok]" unless (3..5).include?(args.length)
if args[-1].downcase == 'ok'
ok = true
args.pop
else
ok = false
end
output_file = args.pop
raise "File '#{output_file}' already exists. specify 'ok' to overwrite it." if File.exist?(output_file) && !ok
template_file = args[0]
raise "File '#{template_file}' does not exist." unless File.exist?(template_file)
translated_values_file = args[1]
raise "File '#{translated_values_file}' does not exist." unless File.exist?(translated_values_file)
partial_translation = args[2]
raise "File '#{partial_translation}' does not exist." if partial_translation && !File.exist?(partial_translation)
return template_file, translated_values_file, partial_translation, output_file
end
def read_template_file()
PropertiesFile.new(@template_file).properties
end
def read_and_merge_translated_values()
lines = IO.readlines(@translated_values_file)
raise "Number of lines in the translated values file (#{lines.size}) does not match the number of properties in the template file (#{@properties_map.size})." unless lines.size == @properties_map.size
count = 0
@properties_map.keys.sort.zip(lines) do |a|
key, value = a
unless @properties_map[key].value.start_with?("@@file")
@properties_map[key].value = value
count += 1
end
end
puts "Merged #{count} translated values."
end
def read_and_merge_partial_translation()
count = 0
if @partial_translation
@partial_map = PropertiesFile.new(@partial_translation).properties
@partial_map.keys.each do |key|
@properties_map[key].value = @partial_map[key].value
count += 1
end
end
puts "Overrode #{count} from partial translation."
end
def write_result()
template_lines = merge_continuation_lines(IO.readlines(@template_file))
File.open(@output_file, 'w') do |f|
template_lines.each do |line|
if line.length == 0 || line[0] == ?# || line[0] == ?!
# copy blank lines, and lines starting with '#' or '!'.
f.puts line
elsif line =~ /(.*?)(\s*[=:]\s*)(.*)/
# key and value are separated by '=' or ':' and optional whitespace.
key = $1.strip
f.puts "#{$1}#{$2}#{@properties_map[key].value}"
else
# No '=' or ':' means that the value was empty.
key = line.strip;
if @properties_map[key]
f.puts "#{key} = #{@properties_map[key].value}"
else
f.puts line
end
end
end
end
end
def merge_continuation_lines(lines)
(lines.size()-1).downto(0) do |i|
if /(.*)\\$/.match(lines[i])
lines[i] = $1 + lines[i+1].lstrip()
lines.delete_at(i+1)
end
end
return lines
end
# ------------------------------------------------------------------------------------
public
# ------------------------------------------------------------------------------------
def initialize(args)
@template_file, @translated_values_file, @partial_translation, @output_file = sanity_check_arguments(args)
end
def process()
@properties_map = read_template_file()
read_and_merge_translated_values()
read_and_merge_partial_translation()
write_result()
puts "Wrote #{@properties_map.length} values to '#{@output_file}'"
end
end
#
#
# ------------------------------------------------------------------------------------
# Standalone calling.
#
# Do this if this program was called from the command line. That is, if the command
# expands to the path of this file.
# ------------------------------------------------------------------------------------
#
if File.expand_path($0) == File.expand_path(__FILE__)
inserter = PropertyInserter.new(ARGV)
inserter.process()
end

89
legacy/utilities/languageSupport/translateValuesInPropertyFile/property_stripper.rb Executable file
View file

@ -0,0 +1,89 @@
#!/usr/bin/ruby
=begin
--------------------------------------------------------------------------------
Read a property file, sort the remainder alphabetically and write their values to
a text file, one value per line.
The idea is that this file could be translated and the result could be used to
create a new property file with property_inserter.rb
On the command line provide the path to the properties file. E.g.:
property_stripper.rb '../../all.properties' output_file
--------------------------------------------------------------------------------
=end
$: << File.dirname(File.expand_path(__FILE__))
require "property_common"
class PropertyStripper
# ------------------------------------------------------------------------------------
private
# ------------------------------------------------------------------------------------
#
# Parse the arguments and complain if they don't make sense.
#
def sanity_check_arguments(args)
raise "usage is: property_stripper.rb <properties_file> <values_output_file> [ok]" unless (2..3).include?(args.length)
if args[2].nil?
ok = false
elsif args[2].downcase == 'ok'
ok = true
else
raise "third argument, if present, must be 'ok'"
end
properties_file = args[0]
raise "File '#{properties_file}' does not exist." unless File.exist?(properties_file)
values_output_file = args[1]
raise "File '#{values_output_file}' already exists. specify 'ok' to overwrite it." if File.exist?(values_output_file) && !ok
return properties_file, values_output_file
end
def read_properties_file(properties_file)
PropertiesFile.new(properties_file).properties
end
def write_values(values_output_file, properties)
File.open(values_output_file, 'w') do |f|
properties.keys.sort.each do |key|
f.puts properties[key].value
end
end
end
# ------------------------------------------------------------------------------------
public
# ------------------------------------------------------------------------------------
def initialize(args)
@properties_file, @values_output_file = sanity_check_arguments(args)
end
def process()
@properties = read_properties_file(@properties_file)
write_values(@values_output_file, @properties)
puts "Wrote #{@properties.length} values to '#{@values_output_file}'"
end
end
#
#
# ------------------------------------------------------------------------------------
# Standalone calling.
#
# Do this if this program was called from the command line. That is, if the command
# expands to the path of this file.
# ------------------------------------------------------------------------------------
#
if File.expand_path($0) == File.expand_path(__FILE__)
stripper = PropertyStripper.new(ARGV)
stripper.process()
end