(PUP-11841) Fix encoding of empty String

In 2077971 empty string literals where
replaced by a call to `String.new` in preparation for moving to
frozen/immutable strings.

However, as stated in the String#new documentation, when no value is
passed to `#new` the returned string has the 'ASCII-8BIT' encoding
instead of the default one (which is assumed to be 'UTF-8' by Puppet).

This cause regressions in some areas of the code, for example when using
the concat module with exported resource built using epp templates, the
incorrect encoding cause the fragment to be misinterpreted and is base64
encoded.  When collected, the base64 representation of the string is
used instead of the actual value of the string, as reported here:
voxpupuli/puppet-bacula#189

Replace calls to `String.new` with `''.dup` which use the current
encoding.  Do not change the few explicit but redundant occurrences of
`String.new.force_encoding('ASCII-8BIT')` (so that the intent is clearly
visible).  Where appropriate, slightly adjust the code for better
readability.

Author: smortex

lib/puppet/application/doc.rb

@@ -152,7 +152,7 @@ def rdoc
   end
 
   def other
-    text = String.new
+    text = ''.dup
     with_contents = options[:references].length <= 1
     exit_code = 0
     require_relative '../../puppet/util/reference'

lib/puppet/face/config.rb

@@ -82,7 +82,7 @@
     end
 
     when_rendering :console do |to_be_rendered|
-      output = String.new
+      output = ''.dup
       if to_be_rendered.keys.length > 1
         to_be_rendered.keys.sort.each do |setting|
           output << "#{setting} = #{to_be_rendered[setting]}\n"

lib/puppet/face/epp.rb

@@ -367,7 +367,7 @@
   end
 
   def dump_parse(source, filename, options, show_filename = true)
-    output = String.new
+    output = ''.dup
     evaluating_parser = Puppet::Pops::Parser::EvaluatingParser::EvaluatingEppParser.new
     begin
       if options[:validate]
@@ -451,7 +451,7 @@ def render_inline(epp_source, compiler, options)
 
   def render_file(epp_template_name, compiler, options, show_filename, file_nbr)
     template_args = get_values(compiler, options)
-    output = String.new
+    output = ''.dup
     begin
       if show_filename && options[:header]
         output << "\n" unless file_nbr == 1

lib/puppet/face/module/list.rb

@@ -74,7 +74,7 @@
       environment     = result[:environment]
       modules_by_path = result[:modules_by_path]
 
-      output = String.new
+      output = ''.dup
 
       warn_unmet_dependencies(environment)
 
@@ -248,7 +248,7 @@ def list_build_tree(list, ancestors=[], parent=nil, params={})
   # Returns a Hash
   #
   def list_build_node(mod, parent, params)
-    str = String.new
+    str = ''.dup
     str << (mod.forge_name ? mod.forge_name.tr('/', '-') : mod.name)
     str << ' (' + colorize(:cyan, mod.version ? "v#{mod.version}" : '???') + ')'
 

lib/puppet/face/parser.rb

@@ -174,7 +174,7 @@
   end
 
   def dump_parse(source, filename, options, show_filename = true)
-    output = String.new
+    output = ''.dup
     evaluating_parser = Puppet::Pops::Parser::EvaluatingParser.new
     begin
       if options[:validate]

lib/puppet/indirector/facts/facter.rb

@@ -105,7 +105,7 @@ def self.setup_external_search_paths(request)
 
   def find_with_options(request)
     options = request.options
-    options_for_facter = String.new
+    options_for_facter = ''.dup
     options_for_facter += options[:user_query].join(' ')
     options_for_facter += " --config #{options[:config_file]}" if options[:config_file]
     options_for_facter += " --show-legacy" if options[:show_legacy]

lib/puppet/indirector/file_bucket_file/file.rb

@@ -58,7 +58,7 @@ def list(request)
       end
       # Setting hash's default value to [], needed by the following loop
       bucket = Hash.new {[]}
-      msg = String.new
+      msg = ''.dup
       # Get all files with mtime between 'from' and 'to'
       Pathname.new(request.options[:bucket_path]).find { |item|
         if item.file? and item.basename.to_s == "paths"

lib/puppet/indirector/indirection.rb

@@ -81,7 +81,7 @@ def expiration
 
   # Generate the full doc string.
   def doc
-    text = String.new
+    text = ''.dup
 
     text << scrub(@doc) << "\n\n" if @doc
 

lib/puppet/module_tool.rb

@@ -70,7 +70,7 @@ def self.is_module_root?(path)
     # Builds a formatted tree from a list of node hashes containing +:text+
     # and +:dependencies+ keys.
     def self.format_tree(nodes, level = 0)
-      str = String.new
+      str = ''.dup
       nodes.each_with_index do |node, i|
         last_node = nodes.length - 1 == i
         deps = node[:dependencies] || []

lib/puppet/network/formats.rb

@@ -156,7 +156,7 @@ def render(datum)
 
     # Simple hash to table
     if datum.is_a?(Hash) && datum.keys.all? { |x| x.is_a?(String) || x.is_a?(Numeric) }
-      output = String.new
+      output = ''.dup
       column_a = datum.empty? ? 2 : datum.map{ |k,v| k.to_s.length }.max + 2
       datum.sort_by { |k,v| k.to_s } .each do |key, value|
         output << key.to_s.ljust(column_a)
@@ -169,7 +169,7 @@ def render(datum)
 
     # Print one item per line for arrays
     if datum.is_a? Array
-      output = String.new
+      output = ''.dup
       datum.each do |item|
         output << item.to_s
         output << "\n"
@@ -227,7 +227,7 @@ def flatten_array(array)
   end
 
   def construct_output(data)
-    output = String.new
+    output = ''.dup
     data.each do |key, value|
       output << "#{key}=#{value}"
       output << "\n"

lib/puppet/network/http/memory_response.rb

@@ -3,7 +3,7 @@ class Puppet::Network::HTTP::MemoryResponse
   attr_reader :code, :type, :body
 
   def initialize
-    @body = String.new
+    @body = ''.dup
   end
 
   def respond_with(code, type, body)

lib/puppet/parameter/value_collection.rb

@@ -31,7 +31,7 @@ def aliasvalue(name, other)
   #
   def doc
     unless defined?(@doc)
-      @doc = String.new
+      @doc = ''.dup
       unless values.empty?
         @doc << "Valid values are "
         @doc << @strings.collect do |value|

lib/puppet/parser/functions.rb

@@ -258,7 +258,7 @@ def self.function(name, environment = Puppet.lookup(:current_environment))
   def self.functiondocs(environment = Puppet.lookup(:current_environment))
     autoloader.delegatee.loadall(environment)
 
-    ret = String.new
+    ret = ''.dup
 
     merged_functions(environment).sort { |a,b| a[0].to_s <=> b[0].to_s }.each do |name, hash|
       ret << "#{name}\n#{"-" * name.to_s.length}\n"

lib/puppet/pops/loader/loader_paths.rb

@@ -88,7 +88,7 @@ def effective_path(typed_name, start_index_in_name)
 
     def typed_name(type, name_authority, relative_path, module_name)
       # Module name is assumed to be included in the path and therefore not added here
-      n = String.new
+      n = ''.dup
       unless extension.empty?
         # Remove extension
         relative_path = relative_path[0..-(extension.length+1)]
@@ -153,7 +153,7 @@ def effective_path(typed_name, start_index_in_name)
     end
 
     def typed_name(type, name_authority, relative_path, module_name)
-      n = String.new
+      n = ''.dup
       n << module_name unless module_name.nil?
       unless extension.empty?
         # Remove extension
@@ -249,7 +249,7 @@ def relative_path
     end
 
     def typed_name(type, name_authority, relative_path, module_name)
-      n = String.new
+      n = ''.dup
       n << module_name unless module_name.nil?
 
       # Remove the file extension, defined as everything after the *last* dot.
@@ -351,7 +351,7 @@ def typed_name(type, name_authority, relative_path, module_name)
       if @init_filenames.include?(relative_path) && !(module_name.nil? || module_name.empty?)
         TypedName.new(type, module_name, name_authority)
       else
-        n = String.new
+        n = ''.dup
         n << module_name unless module_name.nil?
         ext = @extensions.find { |extension| relative_path.end_with?(extension) }
         relative_path = relative_path[0..-(ext.length+1)]

lib/puppet/pops/lookup/explainer.rb

@@ -20,7 +20,7 @@ def to_hash
     end
 
     def explain
-      io = String.new
+      io = ''.dup
       dump_on(io, '', '')
       io
     end

lib/puppet/pops/lookup/hiera_config.rb

@@ -228,7 +228,7 @@ def find_line_matching(regexp, start_line = 1)
       line_number += 1
       next if line_number < start_line
       quote = nil
-      stripped = String.new
+      stripped = ''.dup
       line.each_codepoint do |cp|
         if cp == 0x22 || cp == 0x27 # double or single quote
           if quote == cp

lib/puppet/pops/model/factory.rb

@@ -1112,7 +1112,7 @@ def is_interop_rewriteable?(o)
   end
 
   def self.concat(*args)
-    result = String.new
+    result = ''.dup
     args.each do |e|
       if e.instance_of?(Factory) && e.model_class <= LiteralString
         result << e[KEY_VALUE]

lib/puppet/pops/model/tree_dumper.rb

@@ -21,7 +21,7 @@ def indent
   end
 
   def format(x)
-    result = String.new
+    result = ''.dup
     parts = format_r(x)
     parts.each_index do |i|
       if i > 0

lib/puppet/pops/parser/epp_support.rb

@@ -183,7 +183,7 @@ def scan(skip_leading=false)
       @skip_leading = skip_leading
 
       return nil if scanner.eos?
-      s = String.new
+      s = ''.dup
       until scanner.eos?
         part = @scanner.scan_until(/(<%)|\z/)
         if @skip_leading

lib/puppet/pops/parser/evaluating_parser.rb

@@ -125,7 +125,7 @@ def quote(x)
   # @return [String] The quoted string
   #
   def self.quote(x)
-    escaped = String.new('"')
+    escaped = '"'.dup
     p = nil
     x.each_char do |c|
       case p

lib/puppet/pops/parser/pn_parser.rb

@@ -218,7 +218,7 @@ def consume_identifier(s)
 
   def consume_string
     s = @pos
-    b = String.new
+    b = ''.dup
     loop do
       c = next_cp
       case c

lib/puppet/pops/pn.rb

@@ -20,7 +20,7 @@ def ==(o)
   end
 
   def to_s
-    s = String.new
+    s = ''.dup
     format(nil, s)
     s
   end

lib/puppet/pops/serialization/json_path.rb

@@ -11,7 +11,7 @@ module JsonPath
   #
   # @api private
   def self.to_json_path(path)
-    p = String.new('$')
+    p = '$'.dup
     path.each do |seg|
       if seg.nil?
         p << '[null]'

lib/puppet/pops/time/timespan.rb

@@ -545,7 +545,7 @@ def initialize(format, segments)
       end
 
       def format(timespan)
-        bld = String.new(timespan.negative? ? '-' : '')
+        bld = timespan.negative? ? '-'.dup : ''.dup
         @segments.each { |segment| segment.append_to(bld, timespan) }
         bld
       end
@@ -575,7 +575,7 @@ def regexp
       end
 
       def build_regexp
-        bld = String.new('\A-?')
+        bld = '\A-?'.dup
         @segments.each { |segment| segment.append_regexp(bld) }
         bld << '\z'
         Regexp.new(bld)
@@ -613,7 +613,7 @@ def bad_format_specifier(format, start, position)
 
       def append_literal(bld, codepoint)
         if bld.empty? || !bld.last.is_a?(Format::LiteralSegment)
-          bld << Format::LiteralSegment.new(String.new.concat(codepoint))
+          bld << Format::LiteralSegment.new(''.dup.concat(codepoint))
         else
           bld.last.concat(codepoint)
         end

lib/puppet/pops/types/ruby_generator.rb

@@ -60,7 +60,7 @@ def create_class(obj)
       if cls.nil?
         rp = key.resolved_parent
         parent_class = rp.is_a?(PObjectType) ? rp.implementation_class : Object
-        class_def = String.new
+        class_def = ''.dup
         class_body(key, EMPTY_ARRAY, class_def)
         cls = Class.new(parent_class)
         cls.class_eval(class_def)
@@ -109,7 +109,7 @@ def module_definition(types, comment, *impl_subst)
     end
 
     # Create class definition of all contained types
-    bld = String.new
+    bld = ''.dup
     start_module(common_prefix, comment, bld)
     class_names = []
     names_by_prefix.each_pair do |seg_array, index_and_name_array|

lib/puppet/pops/types/string_converter.rb

@@ -701,7 +701,7 @@ def string_PBooleanType(val_type, val, format_map, indentation)
   # Performs post-processing of literals to apply width and precision flags
   def apply_string_flags(f, literal_str)
     if f.left || f.width || f.prec
-      fmt = String.new('%')
+      fmt = '%'.dup
       fmt << '-' if f.left
       fmt << f.width.to_s if f.width
       fmt << '.' << f.prec.to_s if f.prec
@@ -853,7 +853,7 @@ def puppet_quote(str, enforce_double_quotes = false)
     end
 
     # Assume that the string can be single quoted
-    bld = String.new('\'')
+    bld = "'".dup
     bld.force_encoding(str.encoding)
     escaped = false
     str.each_codepoint do |codepoint|
@@ -879,12 +879,12 @@ def puppet_quote(str, enforce_double_quotes = false)
     # If string ended with a backslash, then that backslash must be escaped
     bld << 0x5c if escaped
 
-    bld << '\''
+    bld << "'"
     bld
   end
 
   def puppet_double_quote(str)
-    bld = String.new('"')
+    bld = '"'.dup
     str.each_codepoint do |codepoint|
       case codepoint
       when 0x09
@@ -940,7 +940,7 @@ def string_PArrayType(val_type, val, format_map, indentation)
 
     case format.format
     when :a, :s, :p
-      buf = String.new
+      buf = ''.dup
       if indentation.breaks?
         buf << "\n"
         buf << indentation.padding
@@ -1055,7 +1055,7 @@ def string_PHashType(val_type, val, format_map, indentation)
 
     when :h, :s, :p
       indentation = indentation.indenting(format.alt? || indentation.is_indenting?)
-      buf = String.new
+      buf = ''.dup
       if indentation.breaks?
         buf << "\n"
         buf << indentation.padding

lib/puppet/pops/types/type_formatter.rb

@@ -50,7 +50,7 @@ def ruby(ref_ctor)
   # @api public
   #
   def string(t)
-    @bld = String.new
+    @bld = ''.dup
     append_string(t)
     @bld
   end
@@ -64,7 +64,7 @@ def string(t)
   #
   # @api public
   def indented_string(t, indent = 0, indent_width = 2)
-    @bld = String.new
+    @bld = ''.dup
     append_indented_string(t, indent, indent_width)
     @bld
   end

lib/puppet/pops/types/types.rb

@@ -1713,7 +1713,7 @@ def self.append_flags_group(rx_string, options)
     if options == 0
       rx_string
     else
-      bld = String.new('(?')
+      bld = '(?'.dup
       bld << 'i' if (options & Regexp::IGNORECASE) != 0
       bld << 'm' if (options & Regexp::MULTILINE) != 0
       bld << 'x' if (options & Regexp::EXTENDED) != 0