#!/usr/bin/ruby

require 'digest'
require 'filesystem'
require 'fileutils'
require 'highline/import'
require 'json'
require 'net/http'
require 'puavo/etc'
require 'puavo/rest-client'
require 'syslog'

def usage
  puts <<'EOF'
puavo-bootserver-sync-images
  options:
    --download-path dir        manage this directory [/images]
    --dry-run                  do nothing but tell the plans
    --force-verify             verify files that are already in place
    --help                     show help
    --space-reserved number    reserved space in bytes [1073741824]

This script manages system images on a bootserver for use with fatclients
and for locally installed clients such as laptops that can download
images/diffs for their use.

This script will delete images and diffs that are not needed unless
those have been locked by creating a file with matching path but
with a ".lock"-suffix added.  System images (normal and backup) are
never removed, also files that have multiple hard links are not removed.

Steps done by the script:
  1. Check image series source URLs from Puavo
  2. Load image series description JSON files from specified URLs
  3. Prioritize files to download and keep.
      a. The image needed by the bootserver itself is prioritized very high.
      b. The newer images and diffs in every series are prioritized higher
         than older images.
      c. Forwards diffs are prioritized higher than backward diffs.
      d. Devices that belong to the schools served by this server affect
         priorizations.  Fatclients want images only and laptops want both
         images and diffs.
  4. Start syncing files, diffs first and then images.
      a. Sync order is files with highest priority first.
      b. If there is not enough space, remove files with the lowest priority.
      c. Image and diff files not belonging to any used series will have
         the lowest priority.
      d. Diffs may be used to construct images from other images.
      e. Create torrent files for diffs after they have been successfully
         installed.
  5. Run /usr/local/lib/puavo-handle-image-changes whenever images
     are installed and at the end to trigger necessary configuration updates.
EOF
end

def log(level, channel, priority, msg, color)
  logmsg = prefixed_logmsg(level, msg)

  Syslog.log(priority, "%s", logmsg)

  outmsg = color  ?  HighLine.new.color(logmsg, color)  :  logmsg
  channel.puts(outmsg)
end

def format_integer(integer)
  num_groups = integer.to_s.chars.to_a.reverse.each_slice(3)
  num_groups.map(&:join).join(',').reverse
end

def prefixed_logmsg(level, msg)
  spacecount = [5, [5-level, 0].max ].min       # (5-level between [0,5])
  arrowcount = [0, [  level, 5].min ].max       # (  level between [0,5])
  "%s %s" % [ (" " * spacecount  +  ">" * arrowcount),
              msg ]
end

def info(level, msg, color=nil)
  log(level, STDOUT, Syslog::LOG_INFO, msg, color)
end

def warning(level, msg)
  log(level, STDERR, Syslog::LOG_WARNING, "WARNING: #{ msg }", HighLine::RED)
end

class Priority
  attr_reader :value

  def initialize(base_priority, adjustments, syncfile)
    @base_priority = base_priority
    @adjustments   = adjustments
    @syncfile      = syncfile

    @value = calculate()
  end

  def calculate
    (@base_priority + @adjustments.sum) \
      / @syncfile.size_penalizer / @syncfile.backward_diff_penalizer
  end

  def calculation
    adjustments_sum_string = @adjustments.join(' + ')
    backwards_penalizer \
      = @syncfile.backward_diff_penalizer != 1 \
          ? " / #{ @syncfile.backward_diff_penalizer } (backwards diff)" \
          : ''
    return sprintf('(%d + %s) / %.3f%s', @base_priority,
                   adjustments_sum_string, @syncfile.size_penalizer,
                   backwards_penalizer)
  end
end

class SyncState
  StatesAndMessages = {
    :missing    => '%s is missing',
    :downloaded => '%s has been downloaded',
    :unverified => '%s is unverified',
    :inplace    => '%s is inplace',
    :deleted    => '%s has been deleted',
    :unknown    => '%s state is unknown',
  }

  def initialize()
    @state = :unknown
  end

  def message(filename)
    return (StatesAndMessages[@state] || StatesAndMessages[:unknown]) \
             % [ filename ]
  end

  def state=(new_state)
    raise "Unsupported state #{ new_state }" \
      unless StatesAndMessages.has_key?(new_state)

    @state = new_state
  end
end

class SyncFile
  attr_accessor :priority
  attr_reader :filename

  def initialize(filename, dir=nil)
    raise 'filename is not set' \
      unless filename && filename.kind_of?(String) && !filename.empty?

    @basedir  = dir || self.class.basedir
    @filename = File.basename(filename)
    @priority = nil
  end

  def all_possible_paths
    return [ download_path,
             partial_unverified_path,
             unverified_path,
             final_path ]
  end

  def backward_diff_penalizer; return 1; end

  def delete
    return self.class.delete_files(all_possible_paths, 3)
  end

  def self.delete_files(filelist, infolevel=4)
    all_deleted_ok = true

    filelist.each do |path|
      next unless File.exist?(path)

      if File.exist?("#{ path }.lock") then
        warning(infolevel, "Not deleting #{ path } because it has .lock")
        all_deleted_ok = false
        next
      end

      # Hardlinked files may be system images or some admin may be doing
      # interesting things.
      if File.stat(path).nlink > 1 then
        warning(infolevel, "Not deleting #{ path } because it has hardlinks")
        all_deleted_ok = false
        next
      end

      if $config[:dry_run] then
        info(infolevel, "Not deleting file at #{ path } (dry-run)",
             HighLine::YELLOW)
        next
      end

      info(infolevel, "Deleting file at #{ path }", HighLine::YELLOW)
      begin
        File.delete(path)
      rescue StandardError => e
        all_deleted_ok = false
        warning(infolevel-1, "Could not delete #{ path }: #{ e.message }")
      end
    end

    return all_deleted_ok
  end

  def download_path
    return "#{ final_path }.partial"
  end

  def final_path
    return "#{ @basedir }/#{ @filename }"
  end

  def inplace?
    return File.exist?(final_path)
  end

  def partial_unverified_path
    return "#{ unverified_path }.partial"
  end

  def unverified_path
    return "#{ final_path }.unverified"
  end
end

class SyncFileWithMetadata < SyncFile
  attr_reader :cksum, :size

  def initialize(cksum, filename, sha256, size, urls)
    super(filename)

    raise 'sha256 is not set' \
      unless sha256 && sha256.kind_of?(String) && !sha256.empty?
    raise 'size is not set' \
      unless size && size.kind_of?(Integer)
    raise 'urls are not set' \
      unless urls && urls.kind_of?(Array) && !urls.empty? \
                  && urls.all? { |url| url.kind_of?(String) }

    @cksum  = cksum
    @sha256 = sha256
    @size   = size
    @urls   = urls

    @in_use        = 0
    @mtime         = nil
    @priority      = nil
    @state         = SyncState.new
    @verified      = false
    @verify_result = false
  end

  def apply_priority_requirements(priority_requirements, device_importance_map)
    adjustments = []

    priority_requirements.each do |device_type, devices_by_syncfiles|
      next unless devices_by_syncfiles.has_key?(@filename)
      devices = devices_by_syncfiles[@filename]
      device_adjustment = devices.count * device_importance_map[device_type]

      unless $log_deduplicator[@filename] then
        msg = sprintf('%s %s required by %d %s%s (adjusting by %d) :: %s',
                      synctype, @filename, devices.count, device_type,
                      (devices.count == 1 ? '' : 's'), device_adjustment,
                      devices.sort.join(' '))
        info(4, msg)
        $log_deduplicator[@filename] = 1
      end
    end

    adjustments.append(0) if adjustments.empty?

    return adjustments
  end

  def delete
    super
    @state.state = :deleted
  end

  def download
    @state.state = :missing

    # Go through all @urls in order and try to download from each in turn
    # (we could also do some load balancing by first choosing one randomly?).
    @urls.each do |url|
      begin
        download_from_url(url)
        return true
      rescue StandardError => e
        warning(2, "Error downloading #{ url }: #{ e.message }")
      end
    end

    return false
  end

  # Download file from specified url and check that the size and the checksum
  # of the downloaded file matches what we expect.  If the file was downloaded
  # successfully and those checks are okay, true is returned, otherwise false
  # is returned.
  def download_from_url(url)
    if $config[:dry_run] then
      info(2, "Not downloading from #{ url } to #{ download_path } (dry-run)")
      return
    end

    uri = URI.parse(url)

    http = Net::HTTP.new(uri.host, uri.port)
    http.ca_file = '/etc/puavo-conf/rootca.pem'
    http.verify_mode = OpenSSL::SSL::VERIFY_PEER
    http.use_ssl = true

    http.request_get(uri.path) do |response|
      case response
      when Net::HTTPNotFound
        raise '404 - Not Found'

      when Net::HTTPOK
        hash = Digest::SHA2.new
        received_size = 0

        info(3, "Downloading from #{ url } to #{ download_path }")

        File.open(download_path, 'w') do |temp_file|
          temp_file.binmode

          progress = 0
          total = response.header['Content-Length'].to_i

          response.read_body do |chunk|
            hash      << chunk
            temp_file << chunk

            received_size += chunk.size
            new_progress = (received_size * 100) / total
            if progress != new_progress then
              progress = new_progress
              msg = 'Download progress: %3d%%' % [progress]
              print("\r" + prefixed_logmsg(2, msg))
            end
          end
        end

        puts '' # output newline after Download progress has gone to 100%

        @state.state = :downloaded

        if received_size != @size then
          begin
            File.delete(download_path)
            info(3, "Deleted #{ download_path }")
          rescue StandardError => e
          end
          raise "Received size (#{ received_size }) did not match" \
                  + " the expected size (#{ @size })"
        end

        if hash.to_s != @sha256 then
          begin
            File.delete(download_path)
            info(3, "Deleted #{ download_path }")
          rescue StandardError => e
          end
          raise "Checksum mismatch for #{ download_path }" \
                  + " (calculated while downloading)," \
                  + " expected #{ @sha256 }, received #{ hash.to_s }"
        end

        info(3, "Checksum verified for #{ download_path }" \
                  + " (calculated while downloading)")
        File.rename(download_path, final_path)
        info(2, "Moved #{ download_path } to #{ final_path }")

        # This can fail and that might be only a temporary error,
        # so ignore errors.
        self.class.run_syncfile_install_hooks(3)

        @state.state = :inplace
        return true
      else
        raise "puavo-rest-client error, received response #{ response.code }"
      end
    end

    raise 'unknown download error'
  end

  # XXX materialize should also generate torrent files?
  def materialize
    verify()         and return true
    patch_from_old() and return true
    download()       and return true

    return false
  end

  def self.run_syncfile_install_hooks(level, timeout=nil)
    # by default do nothing... (subclasses can override this)
  end

  def statemsg
    return @state.message(@filename)
  end

  # Checks if the file in the filesystem has the same sha256 as the metadata.
  # To speed up things, the filesize needs to match before sha256 is
  # calculated.  The results are cached and rechecked only if file mtime
  # changes.
  def verify
    # We can speed things up by assuming that all files in proper place have
    # been verified before (but if $config[:force_verify] is true we will
    # verify anyway).
    if inplace? then
      @state.state = :inplace
      return true unless $config[:force_verify]
      path_to_verify = final_path
    else
      path_to_verify = unverified_path
    end

    if $config[:dry_run] then
      info(3, "Not verifying #{ path_to_verify } (dry-run)")
      return true
    end

    begin
      current_mtime = File.mtime(path_to_verify)
      current_size  = File.size(path_to_verify)
    rescue StandardError => e
      warning(2, "Problem in inspecting #{ path_to_verify }:" \
                   + " #{ e.message }")                       \
        unless e.kind_of?(Errno::ENOENT)
      @state.state = :unverified
      return @verified = @verify_result = false
    end

    if @verified && @mtime && @mtime == current_mtime then
      return @verify_result
    end

    info(3, "Verifying #{ path_to_verify }")

    @verify_result = false
    @state.state = :unverified

    if current_size == @size then
      @mtime = current_mtime
      digest = Digest::SHA2.file(path_to_verify).hexdigest
      @verify_result = (digest == @sha256)
      if @verify_result then
        info(3, "Checksum verified for #{ path_to_verify }")
        if path_to_verify != final_path then
          File.rename(path_to_verify, final_path)
          info(2, "Moved #{ path_to_verify } to #{ final_path }")

          # This can fail and that might be only a temporary error,
          # so ignore errors.
          self.class.run_syncfile_install_hooks(3)
        end
        @state.state = :inplace
      else
        warning(3, "Checksum mismatch for #{ path_to_verify }, " \
                     + " expected #{ @sha256 }, received #{ digest }")
        if path_to_verify == unverified_path then
          File.delete(path_to_verify)
          info(3, "Deleted #{ path_to_verify }")
        else
          # Do not delete in case the file is in final_path... even though
          # the file is probably broken, it is probably better to leave it
          # and try to patch/download a new one in its place.
          # (Verified this might happen if --force-verify option was given.)
        end
      end
    end

    @verified = true

    return @verify_result
  end
end

class Image < SyncFileWithMetadata
  attr_reader :id, :version

  def initialize(cksum, filename, id, sha256, size, urls, version, series)
    super(cksum, filename, sha256, size, urls)

    raise 'id is not set' \
      unless id && id.kind_of?(String) && !id.empty?
    raise 'version is not set' \
      unless version && version.kind_of?(String) && !version.empty?

    @id = id
    @series = series
    @version = version

    @diffs_from_by_version = {}
  end

  def self.basedir; $config[:download_path]; end

  def add_diff(diff)
    return unless diff.targetimage.version == @version

    if diff.baseimage.version != @version then
      @diffs_from_by_version[diff.baseimage.version] = diff
    end
  end

  def diffs
    return @diffs_from_by_version.values
  end

  def diffs_newest_first
    return @diffs_from_by_version.sort.map { |version, diff| diff }.reverse
  end

  def get_priority_adjustments(priority_requirements)
    device_importance_map = {
      :bootserver => 1000000,   # if bootserver wants an image, it must have it
      :fatclient  => 7,         # need image to boot
      :laptop     => 2,         # typically need diffs, not images
    }

    return apply_priority_requirements(priority_requirements[:images],
                                       device_importance_map)
  end

  # Create the current image from an old image by using a patch.  First
  # all the possible diffs are sorted by size and the list is scanned twice:
  # first to check if there is an existing baseimage that has a patch
  # available and if none is found, then the smallest sized diff for an
  # existing baseimage is downloaded.
  def patch_from_old
    inplace_diffs_smallest_first \
      = diffs.select { |diff| diff.inplace? && diff.baseimage.inplace? } \
             .sort_by { |diff| diff.size }

    msg = sprintf('Trying to patch a new image (%s) from existing images' \
                    + ' and diffs', @filename)
    info(3, msg)

    inplace_diffs_smallest_first.each do |diff|
      info(2, "Trying to patch with #{ diff.filename }")
      if diff.apply() && verify() then
        info(1, ' ... patching done.')
        return true
      end
      warning(1, ' ... error when patching.')
    end

    explanation_msg = inplace_diffs_smallest_first.empty? \
                        ? 'no diffs to patch from' \
                        : 'patching attempts failed'
    info(2, "Could not patch from an old image (#{ explanation_msg })",
         HighLine::YELLOW)

    return false
  end

  def self.run_syncfile_install_hooks(level, timeout=nil)
    info(level, 'Running image install hooks')

    all_ok = true

    run_command = lambda do |cmd|
      if system(*cmd) then
        info(level-1, %Q{Command #{ cmd } ran successfully}, HighLine::GREEN)
      else
        all_ok = false
	warning(level-1,
		%Q{Command #{ cmd } returned error code #{ $?.exitstatus }})
      end
    end

    # These are needed to run for each image so that those can be used by
    # netboot (diskless) clients.
    run_command.call([ '/usr/local/lib/puavo-handle-image-changes',
		       '',
		       timeout.to_s ])

    return all_ok
  end


  def set_priorities(base_priority, priority_requirements)
    next_diff_priority = base_priority
    priority_adjustments = get_priority_adjustments(priority_requirements)
    @priority = Priority.new(base_priority, priority_adjustments, self)

    diffs_newest_first.each do |diff|
      diff.set_priorities(next_diff_priority, priority_requirements)
      next_diff_priority = (next_diff_priority/2).floor
    end
  end

  def size_penalizer; (1.0 * @size) / 2**33; end

  def synctype; return 'Image'; end
end

class Diff < SyncFileWithMetadata
  attr_reader :baseimage, :targetimage

  def initialize(cksum, baseimage, targetimage, filename, sha256, size, urls)
    super(cksum, filename, sha256, size, urls)

    @baseimage   = baseimage
    @targetimage = targetimage
    @is_forward  = (baseimage.version < targetimage.version)
  end

  def self.basedir; "#{ $config[:download_path] }/rdiffs"; end

  def apply
    return @baseimage.inplace? && do_patching
  end

  def backward_diff_penalizer
    return @is_forward ? 1 : 5
  end

  def do_patching
    source      = @baseimage.final_path
    target      = @targetimage.unverified_path
    temp_target = @targetimage.partial_unverified_path

    if $config[:dry_run] then
      info(2, "Not patching from #{ source } to #{ temp_target } (dry-run)")
      return true
    end

    info(2, "Patching from #{ source } to #{ temp_target }")

    if !system("rdiff", "patch", source, final_path, temp_target) then
      warning(2, "rdiff failed with exit code: #{ $?.exitstatus }" \
                   + " when using #{ final_path }")
      return false
    end

    info(3, "Patched #{ temp_target } successfully")

    File.rename(temp_target, target)

    info(2, "Moved #{ temp_target } to #{ target }")

    return true
  end

  def get_priority_adjustments(priority_requirements)
    device_importance_map = {
      :bootserver => 100000,    # if bootserver wants a diff, it must have it
      :laptop     => 5,
    }

    return apply_priority_requirements(priority_requirements[:diffs],
                                       device_importance_map)
  end

  def patch_from_old
    # Diffs can not be patched from old/anything, so just return failure.
    return false
  end

  def set_priorities(base_priority, priority_requirements)
    priority_adjustments = get_priority_adjustments(priority_requirements)
    @priority = Priority.new(base_priority, priority_adjustments, self)
  end

  def size_penalizer; (1.0 * @size) / 2**30; end

  def synctype; return 'Diff'; end
end

class Series
  attr_reader :series_name

  def initialize(name)
    @series_name = name

    @by_id = {}
    @by_version = {}
  end

  def add_image(image)
    @by_id[image.id] = @by_version[image.version] = image
  end

  def add_diff(cksum, from_version, to_version, filename, sha256, size, urls)
    baseimage = get_by_version(from_version)
    targetimage = get_by_version(to_version)

    if baseimage && targetimage then
      diff = Diff.new(cksum, baseimage, targetimage, filename, sha256, size,
                      urls)
      targetimage.add_diff(diff)
    end
  end

  def get_by_version(version)
    return @by_version[version]
  end

  def images
    return @by_version.values
  end

  def images_newest_first
    return @by_version.sort.map { |version, image| image }.reverse
  end

  def set_priorities(priority_requirements)
    priority = 1024
    images_newest_first.each do |image|
      image.set_priorities(priority, priority_requirements)
      priority = (priority/4).floor
    end
  end
end

class Torrents
  Torrent_https_port = 873

  def initialize(puavo_etc)
    fqdn = "#{ puavo_etc.hostname }.#{ puavo_etc.domain }"
    @this_server_baseurl = "https://#{ fqdn }:#{ Torrent_https_port }/rdiffs"
    @torrent_paths = {}
  end

  def self.basedir; "#{ $config[:download_path] }/torrents"; end

  def make(syncfile)
    src_path = "#{ syncfile.class.basedir }/#{ syncfile.filename }"
    torrent_filename = "#{ syncfile.filename }.torrent"
    target_path = "#{ self.class.basedir }/#{ torrent_filename }"

    @torrent_paths[target_path] = 1

    return if File.exist?(target_path)

    tmpfile = "#{ target_path }.tmp"

    url = "#{ @this_server_baseurl }/#{ syncfile.filename }"
    cmd_args = [ 'mktorrent', '-o', tmpfile, '-w', url, src_path ]
    fd_redirects = { :in  => '/dev/null',
                     :out => '/dev/null',
                     :err => '/dev/null' }

    if !system(*cmd_args, fd_redirects) then
      raise "command '#{ cmd_args.join(' ') }' returned error"
    end

    File.rename(tmpfile, target_path)

    info(3, "Made a new torrent file #{ target_path }", HighLine::GREEN)
  end

  def remove_unused
    begin
      Dir.glob("#{ self.class.basedir }/*.torrent") do |filepath|
        next if @torrent_paths[filepath]
        begin
          File.delete(filepath)
          info(3, "Removed unused torrent file '#{ filepath }'")
        rescue Errno::ENOENT
        end
      end
    rescue StandardError => e
      warning(4, 'Error in cleaning up unused torrent files')
    end
  end
end

module SeriesControl
  def initialize
    @all_series = nil
  end

  def self.calculate_total_usable_space(files_taking_diskspace)
    free_diskspace = lookup_free_diskspace()
    space_taken_by_files = files_taking_diskspace.values.sum
    usable_space = free_diskspace + space_taken_by_files
    return [ usable_space, 0 ].max
  end

  def self.cleanup_tempfiles
    info(5, 'Cleaning up temporary files')
    glob_patterns = [ File.join(Image.basedir,    '*.partial'),
                      File.join(Image.basedir,    '*.unverified'),
                      File.join(Diff.basedir,     '*.partial'),
                      File.join(Diff.basedir,     '*.unverified'),
                      File.join(Torrents.basedir, '*.tmp') ]
    tempfiles = glob_patterns.map { |pattern| Dir.glob(pattern) }.flatten
    SyncFile.delete_files(tempfiles)
  end

  def self.determine_files_to_sync_and_files_to_delete
    info(5, 'Determining files to sync and files to delete')

    syncfiles_to_delete_priority_order = []
    syncfiles_to_sync                  = []

    files_in_priority_order = get_files_with_highest_priority_first()
    log_files_in_priority_order(files_in_priority_order)

    files_taking_space = lookup_files_taking_space()
    total_usable_space = calculate_total_usable_space(files_taking_space)

    space_available \
      = [ 0, (total_usable_space - $config[:space_reserved]) ].max

    files_in_priority_order.each do |syncfile|
      if syncfile.size > space_available then
        next unless files_taking_space[ syncfile.filename ]
        files_taking_space.delete(syncfile.filename)

        syncfiles_to_delete_priority_order.append(syncfile)
        fmt = 'Marking %s (priority=%.3f size=%s) for deletion as there is' \
                + ' no space available (only %s bytes)'
        msg = sprintf(fmt, syncfile.filename, syncfile.priority.value,
                      format_integer(syncfile.size),
                      format_integer(space_available))
        info(4, msg, HighLine::YELLOW)
        next
      end

      fmt = 'Choosing %s (priority=%.3f size=%s) for sync,' \
              + ' %s bytes are available'
      msg = sprintf(fmt, syncfile.filename, syncfile.priority.value,
                    format_integer(syncfile.size),
                    format_integer(space_available))
      info(4, msg, HighLine::GREEN)
      syncfiles_to_sync.append(syncfile)

      # protected files are not in files_taking_space so we do not subtract
      # those here either
      unless is_protected_file?(syncfile.final_path) then
        space_available -= syncfile.size
      end
    end

    files_to_sync = Hash[ syncfiles_to_sync.map { |sf| [ sf.filename, 1 ] } ]

    files_taking_space.each do |filename, size|
      next if files_to_sync.has_key?(filename)

      msg = sprintf('In case we need more space we may delete %s' \
                      + ' (freeing %s bytes)',
                    filename, format_integer(size))
      info(4, msg, HighLine::YELLOW)

      # setup a simple syncfile only for deletion
      syncfile = filename.match(/\.img$/) \
                   ? SyncFile.new(filename, Image.basedir) \
                   : SyncFile.new(filename, Diff.basedir)
      syncfiles_to_delete_priority_order.append(syncfile)
    end

    # These are from highest priority to lowest.  On deletion we want to
    # delete files with the lowest priority first so we reverse.
    syncfiles_to_delete = syncfiles_to_delete_priority_order.reverse

    return syncfiles_to_sync, syncfiles_to_delete
  end

  def self.ensure_space(needed_space_in_bytes, syncfiles_to_delete)
    until syncfiles_to_delete.empty? do
      free_diskspace = lookup_free_diskspace()
      usable_diskspace = free_diskspace - $config[:space_reserved]
      break unless usable_diskspace < needed_space_in_bytes
      syncfile = syncfiles_to_delete.shift
      msg = sprintf('Deleting %s to make space (need %s bytes,' \
                      + ' got %s bytes (%s - %s)',
                    syncfile.filename, format_integer(needed_space_in_bytes),
                    format_integer(usable_diskspace),
                    format_integer(free_diskspace),
                    format_integer($config[:space_reserved]))
      info(4, msg, HighLine::YELLOW)
      syncfile.delete()
    end

    return syncfiles_to_delete
  end

  # Load series json definition files
  def self.fetch_and_parse_series_data(source_url)
    info(5, "Using image series source url #{ source_url }")

    uri = URI.parse(source_url)

    http = Net::HTTP.new(uri.host, uri.port)
    http.ca_file = '/etc/puavo-conf/rootca.pem'
    http.verify_mode = OpenSSL::SSL::VERIFY_PEER
    http.use_ssl = true

    series_by_name = {}
    http.request_get(uri.path) do |response|
      if !response.kind_of?(Net::HTTPOK) then
        # Do not let a failed fetch slip out as empty series... otherwise
        # temporary download failure might result in removing all images/diffs
        # in this series.
        raise "Error in fetching #{ source_url }," \
                + " response code #{ response.code }"
      end

      series_by_name = parse_series_data( JSON.parse( response.body ) )
    end

    return series_by_name
  end

  def self.get_bootserver_data
    url = "/v3/boot_servers/#{ @etc.hostname }"
    info(5, "Loading bootserver data from puavo-rest url: #{ url }")
    res = @client.get(url)
    return res.parse()
  end

  def self.get_device_bootmode(device)
    # XXX It should be possible to lookup bootmode from device['boot_mode']
    # XXX and it should be either netboot or localboot, but because of
    # XXX bugs in Puavo some localboot devices may be set to 'netboot'.
    # XXX Thus, check out object classes to figure out the correct bootmode.
    object_classes = device['object_classes']
    if !object_classes.kind_of?(Array) then
      errmsg = "Could not determine object classes for #{ hostname }"
      warning(3, errmsg)
      raise errmsg
    end

    return object_classes.include?('puavoNetbootDevice') \
             ? 'netboot'                                 \
             : 'localboot'
  end

  def self.get_devices()
    attributes = %w(current_image hostname object_classes preferred_image
                    puavo_id school_dn)
    res = @client.get('/v3/devices',
                      :params => { :attributes => attributes.join(',') })
    return res.parse()
  end

  def self.get_diff_filename(from_image, to_image)
    # Determine diff filename simply from from_image and to_image strings,
    # without looking at the series data.  We might have diffs that cross
    # from one series to another and those will look like diffs on the
    # originating series (from-image), but should be listed as diffs to the
    # target image.  The diff we return here might not exist in any series
    # but here we do not care about that, at least we could record (elsewhere)
    # what we are missing.
    img_regex = /^(.*?)-([0-9]{4}-[0-9]{2}-[0-9]{2}-[0-9]{6})-(.*?)$/

    from_match = from_image.match(img_regex)
    raise "could not find timestamp in #{ from_image }" unless from_match
    to_match = to_image.match(img_regex)
    raise "could not find timestamp in #{ to_image }" unless to_match

    diff_filename = sprintf('%s-%s--%s-%s.rdiff', from_match[1], from_match[2],
                            to_match[2], from_match[3])
    return diff_filename
  end

  def self.get_priority_requirements_by_devices(bootserver_data)
    info(5, "Checking which images/diffs are needed by bootserver/devices")

    requirements = {
      :diffs => {
        :bootserver => {},
        :laptop     => {},
      },
      :images => {
        :bootserver => {},
        :fatclient  => {},
        :laptop     => {},
      },
    }

    device_list = get_devices()

    bootserver_school_dn_list = Array( bootserver_data['school_dns'] )

    bootserver_current_image   = bootserver_data['current_image']
    bootserver_hostname        = bootserver_data['hostname']
    bootserver_preferred_image = bootserver_data['preferred_image']

    raise 'no bootserver hostname' unless bootserver_hostname

    if bootserver_preferred_image then
      # Image used by the bootserver itself must have a high priority.
      requirements[:images][:bootserver][bootserver_preferred_image] = [
        bootserver_hostname
      ]
    end
    if bootserver_current_image && bootserver_preferred_image \
         && bootserver_current_image != bootserver_preferred_image then
      # The diff required by the bootserver is also very important.
      begin
        diff = get_diff_filename(bootserver_current_image,
                                 bootserver_preferred_image)
        requirements[:diffs][:bootserver][diff] = [ bootserver_hostname ]
      rescue StandardError => e
        warning(4,
                "Error in determining diff filename for #{ hostname }:" \
                  + " #{ e.message }")
      end
    end

    device_list.each do |device|
      current_image   = device['current_image']
      hostname        = device['hostname']
      preferred_image = device['preferred_image']
      puavo_id        = device['puavo_id']
      school_dn       = device['school_dn']

      if !hostname || hostname.empty? then
        warning(4, "There is a device in Puavo (#{ puavo_id }) that does not" \
                     + ' have a hostname')
        next
      end
      if !school_dn || school_dn.empty? then
        warning(4, "Host #{ hostname } is not associated with any school," \
                     + ' ignoring.')
        next
      end

      # Skip hosts that are not served by this bootserver.
      next unless bootserver_school_dn_list.include?(school_dn)

      bootmode = get_device_bootmode(device) rescue nil
      next unless bootmode
      if bootmode == 'netboot' then
        # fatclients are important because they do not work unless they have
        # the image they request
        next unless preferred_image
        filename = '%s.img' % preferred_image
        (requirements[:images][:fatclient][filename] ||= []).append(hostname)
        next
      end

      if current_image && preferred_image \
        && current_image != preferred_image then
          # laptops want diffs
          begin
            diff = get_diff_filename(current_image, preferred_image)
            (requirements[:diffs][:laptop][diff] ||= []).append(hostname)
          rescue StandardError => e
            warning(4,
                    "Error in determining diff filename for #{ hostname }:" \
                      + " #{ e.message }")
          end
      end
    end

    return requirements
  end

  def self.get_files_with_highest_priority_first
    syncfile_list = []
    @all_series.each do |series|
      series.images.each do |img|
        syncfile_list.append(img)
        img.diffs.each do |diff|
          syncfile_list.append(diff)
        end
      end
    end

    # .uniq is used because some images may be listed in multiple series
    # (we should be getting the instance with the highest priority)
    return syncfile_list \
             .sort_by { |syncfile| syncfile.priority.value }.reverse \
             .uniq    { |syncfile| syncfile.filename }
  end

  def self.is_protected_file?(filepath)
    return false unless File.exist?(filepath)

    hardlinked_system_images = [
      File.join(Image.basedir, 'ltsp.img'),
      File.join(Image.basedir, 'ltsp-backup.img'),
    ]

    # We protect files that are our system images or have a hardlink count
    # of more than one or that have a corresponding .lock file so that
    # we do not accidentally delete those.
    return hardlinked_system_images.include?(filepath) \
             || File.stat(filepath).nlink > 1 \
             || File.exist?('%s.lock' % filepath)
  end

  def self.log_files_in_priority_order(files_in_priority_order)
    info(5, 'Listing file priorities:')
    files_in_priority_order.each do |syncfile|
      next unless syncfile.priority.value > 0
      msg = sprintf('%s %s has priority %s = %.3f', syncfile.synctype,
                    syncfile.filename, syncfile.priority.calculation,
                    syncfile.priority.value)
      info(4, msg)
    end
  end

  def self.lookup_files_taking_space
    all_images = Dir.glob("#{ Image.basedir }/*.img")
    all_rdiffs = Dir.glob("#{ Diff.basedir  }/*.rdiff")
    all_files = all_images + all_rdiffs
    files_to_manage = all_files.reject { |fp| is_protected_file?(fp) }

    filesizes_by_filename = {}
    files_to_manage.each do |filepath|
      filename = File.basename(filepath)
      begin
        filesizes_by_filename[filename] = File.stat(filepath).size
      rescue StandardError => e
        raise "Could not determine file #{ filepath } size: #{ e.message }"
      end
    end

    return filesizes_by_filename
  end

  def self.lookup_free_diskspace
    return Sys::Filesystem.stat($config[:download_path]).bytes_available
  end

  def self.parse_series_data(series_data)
    series_by_name = Hash.new

    series_data.each_pair do |series_name, data|
      series = Series.new(series_name)

      data['images'].each do |imagedata|
        image = Image.new(imagedata['cksum'],
                          imagedata['filename'],
                          imagedata['id'],
                          imagedata['sha256'],
                          imagedata['size'],
                          imagedata['urls'],
                          imagedata['version'],
                          series)
        series.add_image(image)
      end

      data['images'].each do |image|
        if image['diffs'] then
          image['diffs'].each do |diffdata|
            series.add_diff(diffdata['cksum'],
                            diffdata['version'],
                            image['version'],
                            diffdata['filename'],
                            diffdata['sha256'],
                            diffdata['size'],
                            diffdata['urls'])
          end
        end
      end

      series_by_name[series_name] = series
    end

    series_by_name
  end

  def self.sync_all_series
    @etc    = PuavoEtc.new
    @client = PuavoRestClient.new :auth => :etc

    bootserver_data = SeriesControl::get_bootserver_data()
    update_all_series(bootserver_data)

    begin
      write_cksums_for_laptops()
    rescue StandardError => e
      warning(5, "Could not write CKSUMS file for laptops: #{ e.message }")
    end

    priority_requirements \
      = get_priority_requirements_by_devices(bootserver_data)

    @all_series.each do |series|
      series.set_priorities(priority_requirements)
    end

    cleanup_tempfiles()

    syncfiles_to_sync, syncfiles_to_delete \
      = determine_files_to_sync_and_files_to_delete()

    if !sync(syncfiles_to_sync, syncfiles_to_delete) then
      return false
    end

    return true
  end

  def self.sync(syncfiles_to_sync, syncfiles_to_delete)
    unless $config[:dry_run] then
      FileUtils.mkdir_p(Image.basedir)
      FileUtils.mkdir_p(Diff.basedir)
      FileUtils.mkdir_p(Torrents.basedir)
    end

    all_syncs_ok = true

    diffs_to_sync  = syncfiles_to_sync.select { |sf| sf.class == Diff  }
    images_to_sync = syncfiles_to_sync.select { |sf| sf.class == Image }

    torrents = Torrents.new(@etc)

    info(5, 'Syncing diffs')
    diffs_to_sync.each do |diff|
      syncfiles_to_delete = ensure_space(diff.size, syncfiles_to_delete)
      info(4, "Syncing #{ diff.filename }")
      if diff.materialize then
        info(3,  "Diff sync OK: #{ diff.statemsg }", HighLine::GREEN)
        begin
          torrents.make(diff)
        rescue StandardError => e
          msg = sprintf('Could not make a torrent for %s: %s', diff.filename,
                        e.message)
          warning(3, msg)
          all_syncs_ok = false
        end
      else
        all_syncs_ok = false
        warning(3, "Diff sync FAILED: #{ diff.statemsg }")
      end
    end

    torrents.remove_unused()

    info(5, 'Syncing images')
    images_to_sync.each do |image|
      syncfiles_to_delete = ensure_space(image.size, syncfiles_to_delete)
      info(4, "Syncing #{ image.filename }")
      if image.materialize then
        info(3, "Image sync OK: #{ image.statemsg }", HighLine::GREEN)
      else
        all_syncs_ok = false
        warning(3, "Image sync FAILED: #{ image.statemsg }")
      end
    end

    return all_syncs_ok
  end

  def self.update_all_series(bootserver_data)
    all_series_list = []
    series_urls = Array(bootserver_data['image_series_source_urls'])

    if series_urls.empty? then
      info(5, "No series urls were defined")
    else
      series_urls.each do |series_url|
        all_series_list << fetch_and_parse_series_data(series_url)
      end
    end

    all_series = all_series_list.reduce(&:merge) || {}

    if all_series.empty? then
      raise "No series information, can not do anything"
    end

    @all_series = all_series.values
  end

  def self.write_cksums_for_laptops
    syncfiles = {}

    @all_series.each do |series|
      series.images.each do |image|
        next unless image.cksum && image.filename && image.size
        syncfiles[ image.filename ] = image
        image.diffs.each { |diff| syncfiles[ diff.filename ] = diff }
      end
    end

    cksums_file_contents \
      = syncfiles.values \
                 .map { |sf| [sf.cksum, sf.size, sf.filename].join(' ') } \
                 .uniq.map { |s| "#{ s }\n" }.join

    raise "Download path #{ $config[:download_path] } is not a directory" \
      unless File.directory?($config[:download_path])

    cksums_file = "#{ $config[:download_path] }/CKSUMS"

    if $config[:dry_run] then
      info(5, "Not writing #{ cksums_file } (dry-run)")
      return
    end

    tmpfile = "#{ cksums_file }.tmp"
    File.open(tmpfile, 'w') { |f| f.write cksums_file_contents }
    File.rename(tmpfile, cksums_file)

    info(5, "Wrote #{ cksums_file } (this is for legacy updates)")
  end
end

exitstatus = 0

image_updates_lockfile = nil

$log_deduplicator = {}

script_name = File.basename(__FILE__)
Syslog.open(script_name, Syslog::LOG_CONS)

$config = {
  :download_path  => '/images',
  :dry_run        => false,
  :force_verify   => false,
  :space_reserved => 2**30,         # one gigabyte
}

begin
  parser = OptionParser.new do |opts|
    opts.on('--download-path',
            'The target directory for images/rdiffs') do |path|
      $config[:download_path] = path
    end
    opts.on('--dry-run',
            'Only show what should be done, yet do nothing') do
      $config[:dry_run] = true
    end
    opts.on('--force-verify', 'Force verification of each image/rdiff') do
      $config[:force_verify] = true
    end
    opts.on('--space-reserved',
            'The amount of space reserved in bytes') do |value|
      unless value.kind_of?(String) && value.match(/\A\d+\z/) then
        usage
        exit 1
      end
      $config[:space_reserved] = Integer(value)
    end
    opts.on_tail('--help', 'Show help') do
      usage
      exit 0
    end
  end
  parser.parse!

  # use a common lock with puavo-install-and-update-ltspimages
  # (only one instance of these two programs should be running)
  image_updates_lockfile_path \
    = "#{ $config[:download_path] }/.image_updates_lock"
  image_updates_lockfile = File.open(image_updates_lockfile_path, 'a+')
  if !image_updates_lockfile.flock(File::LOCK_NB|File::LOCK_EX) then
    raise "Could not get a lock on #{ image_updates_lockfile_path }"
  end

  if !SeriesControl::sync_all_series() then
    warning(5, 'Errors when syncing')
    exitstatus = 2
  end

  if !Image.run_syncfile_install_hooks(5, 43200) then
    exitstatus = 1 if exitstatus == 0
  end
rescue StandardError => e
  warning(5, "#{ e.message } / #{ e.backtrace.join(' / ') }")
  exitstatus = 1
end

Syslog.close()

image_updates_lockfile.close if image_updates_lockfile

exit(exitstatus)
