/**
 * Orthanc - A Lightweight, RESTful DICOM Store
 * Copyright (C) 2012-2016 Sebastien Jodogne, Medical Physics
 * Department, University Hospital of Liege, Belgium
 * Copyright (C) 2017-2023 Osimis S.A., Belgium
 * Copyright (C) 2024-2026 Orthanc Team SRL, Belgium
 * Copyright (C) 2021-2026 Sebastien Jodogne, ICTEAM UCLouvain, Belgium
 *
 * This program is free software: you can redistribute it and/or
 * modify it under the terms of the GNU General Public License as
 * published by the Free Software Foundation, either version 3 of the
 * License, or (at your option) any later version.
 * 
 * This program is distributed in the hope that it will be useful, but
 * WITHOUT ANY WARRANTY; without even the implied warranty of
 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
 * General Public License for more details.
 *
 * You should have received a copy of the GNU General Public License
 * along with this program. If not, see <http://www.gnu.org/licenses/>.
 **/


#include "../PrecompiledHeadersServer.h"
#include "ArchiveJob.h"

#include "../../../OrthancFramework/Sources/Cache/SharedArchive.h"
#include "../../../OrthancFramework/Sources/Compression/HierarchicalZipWriter.h"
#include "../../../OrthancFramework/Sources/Constants.h"
#include "../../../OrthancFramework/Sources/DataSource/DicomSequentialReader.h"
#include "../../../OrthancFramework/Sources/DicomParsing/DicomDirWriter.h"
#include "../../../OrthancFramework/Sources/DicomParsing/FromDcmtkBridge.h"
#include "../../../OrthancFramework/Sources/Logging.h"
#include "../../../OrthancFramework/Sources/MultiThreading/BlockingSharedMessageQueue.h"
#include "../../../OrthancFramework/Sources/MultiThreading/Semaphore.h"
#include "../../../OrthancFramework/Sources/OrthancException.h"
#include "../../../OrthancFramework/Sources/SerializationToolbox.h"
#include "../OrthancConfiguration.h"
#include "../ServerContext.h"
#include "../SimpleInstanceOrdering.h"

#include <cassert>
#include <stdio.h>
#include <boost/range/algorithm/count.hpp>

#if defined(_MSC_VER)
#define snprintf _snprintf
#endif

static const char* const MEDIA_IMAGES_FOLDER = "IMAGES"; 
static const char* const KEY_DESCRIPTION = "Description";
static const char* const KEY_INSTANCES_COUNT = "InstancesCount";
static const char* const KEY_UNCOMPRESSED_SIZE_MB = "UncompressedSizeMB";
static const char* const KEY_ARCHIVE_SIZE_MB = "ArchiveSizeMB";
static const char* const KEY_UNCOMPRESSED_SIZE = "UncompressedSize";
static const char* const KEY_ARCHIVE_SIZE = "ArchiveSize";
static const char* const KEY_TRANSCODE = "Transcode";
static const char* const KEY_ALLOW_UTF8 = "Utf8";
static const char* const KEY_LOSSY_QUALITY = "LossyQuality";


namespace Orthanc
{
  static bool IsZip64Required(uint64_t uncompressedSize,
                              unsigned int countInstances)
  {
    static const uint64_t      SAFETY_MARGIN = 64 * MEGABYTE;  // Should be large enough to hold DICOMDIR
    static const unsigned int  FILES_MARGIN = 10;

    /**
     * Determine whether ZIP64 is required. Original ZIP format can
     * store up to 2GB of data (some implementation supporting up to
     * 4GB of data), and up to 65535 files.
     * https://en.wikipedia.org/wiki/Zip_(file_format)#ZIP64
     **/

    const bool isZip64 = (uncompressedSize >= 2 * GIGABYTE - SAFETY_MARGIN ||
                          countInstances >= 65535 - FILES_MARGIN);

    LOG(INFO) << "Creating a ZIP file with " << countInstances << " files of size "
              << Toolbox::GetHumanFileSize(uncompressedSize) << " using the "
              << (isZip64 ? "ZIP64" : "ZIP32") << " file format";

    return isZip64;
  }



  // This enum defines specific resource types to be used when exporting the archive.
  // It defines if we should use the PatientInfo from the Patient or from the Study.
  enum ArchiveResourceType
  {
    ArchiveResourceType_Patient = 0,
    ArchiveResourceType_PatientInfoFromStudy = 1,
    ArchiveResourceType_Study = 2,
    ArchiveResourceType_Series = 3,
    ArchiveResourceType_Instance = 4
  };

  ResourceType GetResourceIdType(ArchiveResourceType type)
  {
    switch (type)
    {
      case ArchiveResourceType_Patient:
        return ResourceType_Patient;
      case ArchiveResourceType_PatientInfoFromStudy: // get the Patient tags from the Study id
        return ResourceType_Study;
      case ArchiveResourceType_Study:
        return ResourceType_Study;
      case ArchiveResourceType_Series:
        return ResourceType_Series;
      case ArchiveResourceType_Instance:
        return ResourceType_Instance;
      default:
        throw OrthancException(ErrorCode_ParameterOutOfRange);
    }
  }

  ResourceType GetResourceLevel(ArchiveResourceType type)
  {
    switch (type)
    {
      case ArchiveResourceType_Patient:
        return ResourceType_Patient;
      case ArchiveResourceType_PatientInfoFromStudy: // this is actually the same level as the Patient
        return ResourceType_Patient;
      case ArchiveResourceType_Study:
        return ResourceType_Study;
      case ArchiveResourceType_Series:
        return ResourceType_Series;
      case ArchiveResourceType_Instance:
        return ResourceType_Instance;
      default:
        throw OrthancException(ErrorCode_ParameterOutOfRange);
    }
  }

  ArchiveResourceType GetArchiveResourceType(ResourceType type)
  {
    switch (type)
    {
      case ResourceType_Patient:
        return ArchiveResourceType_Patient;
      case ResourceType_Study:
       return ArchiveResourceType_PatientInfoFromStudy;
      case ResourceType_Series:
        return ArchiveResourceType_Series;
      case ResourceType_Instance:
        return ArchiveResourceType_Instance;
      default:
        throw OrthancException(ErrorCode_ParameterOutOfRange);
    }
  }

  ArchiveResourceType GetChildResourceType(ArchiveResourceType type)
  {
    switch (type)
    {
      case ArchiveResourceType_Patient:
      case ArchiveResourceType_PatientInfoFromStudy:
        return ArchiveResourceType_Study;

      case ArchiveResourceType_Study:
        return ArchiveResourceType_Series;
        
      case ArchiveResourceType_Series:
        return ArchiveResourceType_Instance;

      default:
        throw OrthancException(ErrorCode_ParameterOutOfRange);
    }
  }


  class ArchiveJob::ResourceIdentifiers : public boost::noncopyable
  {
  private:
    ResourceType   level_;
    std::string    patient_;
    std::string    study_;
    std::string    series_;
    std::string    instance_;

    static void GoToParent(ServerIndex& index,
                           std::string& current)
    {
      std::string tmp;

      if (index.LookupParent(tmp, current))
      {
        current = tmp;
      }
      else
      {
        throw OrthancException(ErrorCode_UnknownResource);
      }
    }


  public:
    ResourceIdentifiers(ServerIndex& index,
                        const std::string& publicId)
    {
      if (!index.LookupResourceType(level_, publicId))
      {
        throw OrthancException(ErrorCode_UnknownResource);
      }

      std::string current = publicId;;
      switch (level_)  // Do not add "break" below!
      {
        case ResourceType_Instance:
          instance_ = current;
          GoToParent(index, current);
            
        case ResourceType_Series:
          series_ = current;
          GoToParent(index, current);

        case ResourceType_Study:
          study_ = current;
          GoToParent(index, current);

        case ResourceType_Patient:
          patient_ = current;
          break;

        default:
          THROW_WITH_FILE_AND_LINE_INFO(ErrorCode_InternalError);
      }
    }

    ResourceType GetLevel() const
    {
      return level_;
    }

    const std::string& GetIdentifier(ResourceType level) const
    {
      // Some sanity check to ensure enumerations are not altered
      assert(ResourceType_Patient < ResourceType_Study);
      assert(ResourceType_Study < ResourceType_Series);
      assert(ResourceType_Series < ResourceType_Instance);

      if (level > level_)
      {
        THROW_WITH_FILE_AND_LINE_INFO(ErrorCode_InternalError);
      }

      switch (level)
      {
        case ResourceType_Patient:
          return patient_;

        case ResourceType_Study:
          return study_;

        case ResourceType_Series:
          return series_;

        case ResourceType_Instance:
          return instance_;

        default:
          THROW_WITH_FILE_AND_LINE_INFO(ErrorCode_InternalError);
      }
    }
  };


  class ArchiveJob::IArchiveVisitor : public boost::noncopyable
  {
  public:
    virtual ~IArchiveVisitor()
    {
    }

    virtual void Open(ArchiveResourceType level,
                      const std::string& publicId) = 0;

    virtual void Close() = 0;

    virtual void AddInstance(const std::string& instanceId,
                             uint32_t index,
                             const FileInfo& fileInfo) = 0;
  };


  class ArchiveJob::ArchiveIndex : public boost::noncopyable
  {
  private:
    struct Instance
    {
      std::string  id_;
      uint32_t     index_;
      FileInfo     fileInfo_;

      Instance(const std::string& id,
               uint32_t index,
               const FileInfo& fileInfo) :
        id_(id),
        index_(index),
        fileInfo_(fileInfo)
      {
      }
    };

    // A "NULL" value for ArchiveIndex indicates a non-expanded node
    typedef std::map<std::string, ArchiveIndex*>   Resources;

    ArchiveResourceType  level_;
    Resources            resources_;   // Only at patient/study/series level
    std::list<Instance>  instances_;   // Only at instance level


    void AddResourceToExpand(const std::string& id)
    {
      assert(level_ != ArchiveResourceType_Instance);
      resources_[id] = NULL;
    }

    void AddInstance(const std::string& id,
                     uint32_t indexInSeries,
                     const FileInfo& fileInfo)
    {
      assert(level_ == ArchiveResourceType_Instance);
      instances_.push_back(Instance(id, indexInSeries, fileInfo));
    }


  public:
    explicit ArchiveIndex(ArchiveResourceType level) :
      level_(level)
    {
    }

    ~ArchiveIndex()
    {
      for (Resources::iterator it = resources_.begin();
           it != resources_.end(); ++it)
      {
        delete it->second;
      }
    }


    void Add(ServerIndex& index,
             const ResourceIdentifiers& resource)
    {
      const std::string& id = resource.GetIdentifier(GetResourceIdType(level_));
      Resources::iterator previous = resources_.find(id);

      if (level_ == ArchiveResourceType_Instance)
      {
        std::string strIndexInSeries;
        uint32_t indexInSeries = 0;
        FileInfo fileInfo;
        int64_t revisionNotUsed;
        
        if (index.LookupMetadata(strIndexInSeries, id, ResourceType_Instance, MetadataType_Instance_IndexInSeries))
        {
          SerializationToolbox::ParseUnsignedInteger32(indexInSeries, strIndexInSeries);
        }

        if (index.LookupAttachment(fileInfo, revisionNotUsed, ResourceType_Instance, id, FileContentType_Dicom))
        {
          AddInstance(id, indexInSeries, fileInfo);
        }
      }
      else if (resource.GetLevel() == GetResourceLevel(level_))
      {
        // Mark this resource for further expansion
        if (previous != resources_.end())
        {
          delete previous->second;
        }

        resources_[id] = NULL;
      }
      else if (previous == resources_.end())
      {
        // This is the first time we meet this resource
        std::unique_ptr<ArchiveIndex> child(new ArchiveIndex(GetChildResourceType(level_)));
        child->Add(index, resource);
        resources_[id] = child.release();
      }
      else if (previous->second != NULL)
      {
        previous->second->Add(index, resource);
      }
      else
      {
        // Nothing to do: This item is marked for further expansion
      }
    }


    void Expand(ServerIndex& index)
    {
      if (level_ == ArchiveResourceType_Instance)
      {
        // Expanding an instance node makes no sense
        return;
      }

      for (Resources::iterator it = resources_.begin();
           it != resources_.end(); ++it)
      {
        if (it->second == NULL)
        {
          // This resource is marked for expansion
          std::unique_ptr<ArchiveIndex> child(new ArchiveIndex(GetChildResourceType(level_)));

          if (level_ == ArchiveResourceType_Series)
          {
            // Instances ordering is important !  
            // From 1.12.6, when possible, the id in the filename will match the index in series.
            // Only if there are duplicate index in series, we'll use a simple counter
            SimpleInstanceOrdering orderedInstances(index, it->first);

            for (size_t i = 0; i < orderedInstances.GetInstancesCount(); ++i)
            {
              child->AddInstance(orderedInstances.GetInstanceId(i), orderedInstances.GetInstanceIndexInSeries(i), orderedInstances.GetInstanceFileInfo(i));
            }
          }
          else
          {
            std::list<std::string> children;
            index.GetChildren(children, GetResourceLevel(level_), it->first);

            for (std::list<std::string>::const_iterator 
                  it2 = children.begin(); it2 != children.end(); ++it2)
            {
              child->AddResourceToExpand(*it2);
            }
          }

          it->second = child.release();
        }

        assert(it->second != NULL);
        it->second->Expand(index);
      }        
    }


    void Apply(IArchiveVisitor& visitor) const
    {
      if (level_ == ArchiveResourceType_Instance)
      {
        for (std::list<Instance>::const_iterator 
               it = instances_.begin(); it != instances_.end(); ++it)
        {
          visitor.AddInstance(it->id_,  it->index_, it->fileInfo_);
        }          
      }
      else
      {
        for (Resources::const_iterator it = resources_.begin();
             it != resources_.end(); ++it)
        {
          assert(it->second != NULL);  // There must have been a call to "Expand()"
          visitor.Open(level_, it->first);
          it->second->Apply(visitor);
          visitor.Close();
        }
      }
    }
  };


  class ArchiveJob::IZipCommand : public boost::noncopyable
  {
  public:
    virtual ~IZipCommand()
    {
    }

    virtual void Preload(DicomSequentialReader& instancesLoader) const = 0;

    // In the "archive" flavor (without DICOMDIR), dicomDir is NULL
    // In the "media" flavor (with DICOMDIR), dicomDir is not NULL
    virtual void Execute(HierarchicalZipWriter& writer,
                         DicomSequentialReader& instancesLoader,
                         DicomDirWriter* dicomDir) const = 0;
  };


  class ArchiveJob::ZipCommands : public boost::noncopyable
  {
  private:
    class OpenDirectoryCommand : public IZipCommand
    {
    private:
      std::string   filename_;

    public:
      explicit OpenDirectoryCommand(const std::string& filename) :
        filename_(filename)
      {
      }

      virtual void Preload(DicomSequentialReader& instancesLoader) const ORTHANC_OVERRIDE
      {
      }

      virtual void Execute(HierarchicalZipWriter& writer,
                           DicomSequentialReader& instancesLoader,
                           DicomDirWriter* dicomDir) const ORTHANC_OVERRIDE
      {
        writer.OpenDirectory(filename_);
      }
    };


    class CloseDirectoryCommand : public IZipCommand
    {
    public:
      virtual void Preload(DicomSequentialReader& instancesLoader) const ORTHANC_OVERRIDE
      {
      }

      virtual void Execute(HierarchicalZipWriter& writer,
                           DicomSequentialReader& instancesLoader,
                           DicomDirWriter* dicomDir) const ORTHANC_OVERRIDE
      {
        writer.CloseDirectory();
      }
    };


    class WriteInstanceCommand : public IZipCommand
    {
    private:
      std::string   filename_;
      std::string   instanceId_;
      FileInfo      fileInfo_;
      std::string   dicomDirFolder_;

    public:
      WriteInstanceCommand(const std::string& filename,
                           const std::string& instanceId,
                           const FileInfo& fileInfo,
                           const std::string& dicomDirFolder /* in practice, this is the constant "IMAGES" */) :
        filename_(filename),
        instanceId_(instanceId),
        fileInfo_(fileInfo),
        dicomDirFolder_(dicomDirFolder)
      {
        assert(dicomDirFolder.empty() ||
               dicomDirFolder == MEDIA_IMAGES_FOLDER);
      }

      virtual void Preload(DicomSequentialReader& instancesLoader) const ORTHANC_OVERRIDE
      {
        instancesLoader.Submit(fileInfo_);
      }

      virtual void Execute(HierarchicalZipWriter& writer,
                           DicomSequentialReader& instancesLoader,
                           DicomDirWriter* dicomDir) const ORTHANC_OVERRIDE
      {
        if (!instancesLoader.HasNext())
        {
          throw OrthancException(ErrorCode_InternalError);
        }
        else
        {
          std::unique_ptr<DicomSequentialReader::Item> next;

          try
          {
            LOG(INFO) << "Adding instance " << instanceId_ << " in zip";
            next.reset(instancesLoader.Next());
          }
          catch (OrthancException& e)
          {
            LOG(WARNING) << "An instance was removed after the job was issued: " << instanceId_;
            return;
          }

          const IMemoryBuffer& dicom = next->GetRawMemoryBuffer();

          writer.OpenFile(filename_);
          writer.Write(dicom.GetData(), dicom.GetSize());

          if (dicomDir != NULL)
          {
            dicomDir->Add(dicomDirFolder_, filename_, next->GetParsedDicomFile());
          }
        }
      }
    };
      
    std::deque<IZipCommand*>  commands_;
    uint64_t                  uncompressedSize_;
    unsigned int              instancesCount_;
      
  public:
    explicit ZipCommands() :
      uncompressedSize_(0),
      instancesCount_(0)
    {
    }
      
    ~ZipCommands()
    {
      for (std::deque<IZipCommand*>::iterator it = commands_.begin();
           it != commands_.end(); ++it)
      {
        assert(*it != NULL);
        delete *it;
      }
    }

    size_t GetSize() const
    {
      return commands_.size();
    }

    unsigned int GetInstancesCount() const
    {
      return instancesCount_;
    }

    uint64_t GetUncompressedSize() const
    {
      return uncompressedSize_;
    }

    void AddOpenDirectory(const std::string& filename)
    {
      commands_.push_back(new OpenDirectoryCommand(filename));
    }

    void AddCloseDirectory()
    {
      commands_.push_back(new CloseDirectoryCommand);
    }

    void AddWriteInstance(const std::string& filename,
                          const std::string& instanceId,
                          const FileInfo& fileInfo,
                          const std::string& dicomDirFolder)
    {
      commands_.push_back(new WriteInstanceCommand(filename, instanceId, fileInfo, dicomDirFolder));
      instancesCount_ ++;
      uncompressedSize_ += fileInfo.GetUncompressedSize();
    }

    bool IsZip64() const
    {
      return IsZip64Required(GetUncompressedSize(), GetInstancesCount());
    }

    const IZipCommand& GetCommand(size_t index) const
    {
      if (index < commands_.size())
      {
        assert(commands_[index] != NULL);
        return *commands_[index];
      }
      else
      {
        throw OrthancException(ErrorCode_ParameterOutOfRange);
      }
    }
  };
    
    

  class ArchiveJob::ArchiveIndexVisitor : public IArchiveVisitor
  {
  private:
    ZipCommands&    commands_;
    ServerContext&  context_;
    char            instanceFormat_[24];
    unsigned int    counter_;

    static std::string GetTag(const DicomMap& tags,
                              const DicomTag& tag)
    {
      const DicomValue* v = tags.TestAndGetValue(tag);
      if (v != NULL &&
          !v->IsBinary() &&
          !v->IsNull())
      {
        return v->GetContent();
      }
      else
      {
        return "";
      }
    }

  public:
    ArchiveIndexVisitor(ZipCommands& commands,
                        ServerContext& context) :
      commands_(commands),
      context_(context),
      counter_(0)
    {
      if (commands.GetSize() != 0)
      {
        throw OrthancException(ErrorCode_BadSequenceOfCalls);
      }
        
      snprintf(instanceFormat_, sizeof(instanceFormat_) - 1, "%%08d.dcm");
    }

    virtual void Open(ArchiveResourceType level,
                      const std::string& publicId) ORTHANC_OVERRIDE
    {
      std::string path;
      std::vector<std::string> pathParts;

      DicomMap tags;
      ResourceType resourceIdLevel = GetResourceIdType(level);
      ResourceType interestLevel = (level == ArchiveResourceType_PatientInfoFromStudy ? ResourceType_Patient : resourceIdLevel);

      if (context_.GetIndex().GetMainDicomTags(tags, publicId, resourceIdLevel, interestLevel))
      {
        switch (level)
        {
          case ArchiveResourceType_Patient:
          case ArchiveResourceType_PatientInfoFromStudy:
          {
            pathParts.push_back(GetTag(tags, DICOM_TAG_PATIENT_ID));
            std::string patientName = GetTag(tags, DICOM_TAG_PATIENT_NAME);
            if (!patientName.empty())
            {
              pathParts.push_back(patientName);
            }
          }; break;
          case ArchiveResourceType_Study:
          {
            std::string accessionNumber = GetTag(tags, DICOM_TAG_ACCESSION_NUMBER);
            std::string studyDescription = GetTag(tags, DICOM_TAG_STUDY_DESCRIPTION);
            if (!accessionNumber.empty())
            {
              pathParts.push_back(accessionNumber);
            }
            if (!studyDescription.empty())
            {
              pathParts.push_back(studyDescription);
            }
          }; break;
          case ArchiveResourceType_Series:
          {
            std::string seriesNumber = GetTag(tags, DICOM_TAG_SERIES_NUMBER);
            std::string modality = GetTag(tags, DICOM_TAG_MODALITY);
            std::string seriesDescription = GetTag(tags, DICOM_TAG_SERIES_DESCRIPTION);
            
            if (!seriesNumber.empty())
            {
              pathParts.push_back(seriesNumber);
            }
            
            if (!modality.empty())
            {
              pathParts.push_back(modality);
            }
            
            if (!seriesDescription.empty())
            {
              pathParts.push_back(seriesDescription);
            }
            
            if (modality.size() == 0)
            {
              snprintf(instanceFormat_, sizeof(instanceFormat_) - 1, "%%08d.dcm");
            }
            else if (modality.size() == 1)
            {
              snprintf(instanceFormat_, sizeof(instanceFormat_) - 1, "%c%%07d.dcm", 
                       toupper(modality[0]));
            }
            else if (modality.size() >= 2)
            {
              snprintf(instanceFormat_, sizeof(instanceFormat_) - 1, "%c%c%%06d.dcm", 
                       toupper(modality[0]), toupper(modality[1]));
            }

            counter_ = 0;
          }; break;
          default:
            THROW_WITH_FILE_AND_LINE_INFO(ErrorCode_InternalError);
        }
      }

      Toolbox::JoinStrings(path, pathParts, " - ");
      path =  Toolbox::StripSpaces(path);

      if (path.empty() 
          || (static_cast<size_t>(boost::count(path, '^')) == path.size()))  // this happens with non ASCII patient names: only the '^' remains and this is not a valid zip folder name
      {
        path = std::string("Unknown ") + EnumerationToString(GetResourceLevel(level));
      }

      commands_.AddOpenDirectory(path);
    }

    virtual void Close() ORTHANC_OVERRIDE
    {
      commands_.AddCloseDirectory();
    }

    virtual void AddInstance(const std::string& instanceId,
                             uint32_t index,
                             const FileInfo& fileInfo) ORTHANC_OVERRIDE
    {
      char filename[24];
      snprintf(filename, sizeof(filename) - 1, instanceFormat_, index);

      commands_.AddWriteInstance(filename, instanceId, fileInfo, "" /* no DICOMDIR */);
    }
  };

    
  class ArchiveJob::MediaIndexVisitor : public IArchiveVisitor
  {
  private:
    ZipCommands&    commands_;
    unsigned int    counter_;

  public:
    explicit MediaIndexVisitor(ZipCommands& commands) :
      commands_(commands),
      counter_(0)
    {
    }

    virtual void Open(ArchiveResourceType level,
                      const std::string& publicId) ORTHANC_OVERRIDE
    {
    }

    virtual void Close() ORTHANC_OVERRIDE
    {
    }

    virtual void AddInstance(const std::string& instanceId,
                             uint32_t indexNotUsed,
                             const FileInfo& fileInfo) ORTHANC_OVERRIDE
    {
      // "DICOM restricts the filenames on DICOM media to 8
      // characters (some systems wrongly use 8.3, but this does not
      // conform to the standard)."
      std::string filename = "IM" + boost::lexical_cast<std::string>(counter_);
      commands_.AddWriteInstance(filename, instanceId, fileInfo, MEDIA_IMAGES_FOLDER);

      counter_ ++;
    }
  };


  class ArchiveJob::ZipWriterIterator : public boost::noncopyable
  {
  private:
    ServerContext&                          context_;
    ZipCommands                             commands_;
    std::unique_ptr<HierarchicalZipWriter>  zip_;
    std::unique_ptr<DicomDirWriter>         dicomDir_;
    bool                                    isMedia_;
    bool                                    isStream_;
    bool                                    allowUtf8_;

  public:
    ZipWriterIterator(ServerContext& context,
                      ArchiveIndex& archive,
                      bool isMedia,
                      bool enableExtendedSopClass,
                      bool allowUtf8) :
      context_(context),
      isMedia_(isMedia),
      isStream_(false),
      allowUtf8_(allowUtf8)
    {
      if (isMedia)
      {
        MediaIndexVisitor visitor(commands_);
        archive.Expand(context.GetIndex());

        commands_.AddOpenDirectory(MEDIA_IMAGES_FOLDER);        
        archive.Apply(visitor);
        commands_.AddCloseDirectory();

        dicomDir_.reset(new DicomDirWriter);
        dicomDir_->EnableExtendedSopClass(enableExtendedSopClass);
      }
      else
      {
        ArchiveIndexVisitor visitor(commands_, context);
        archive.Expand(context.GetIndex());
        archive.Apply(visitor);
      }
    }

    void SetOutputFile(const boost::filesystem::path& path)
    {
      if (zip_.get() == NULL)
      {
        zip_.reset(new HierarchicalZipWriter(path));
        zip_->SetZip64(commands_.IsZip64());
        zip_->SetAllowUtf8(allowUtf8_);
        isStream_ = false;
      }
      else
      {
        throw OrthancException(ErrorCode_BadSequenceOfCalls);
      }
    }

    void AcquireOutputStream(ZipWriter::IOutputStream* output)
    {
      std::unique_ptr<ZipWriter::IOutputStream> protection(output);

      if (zip_.get() == NULL)
      {
        zip_.reset(new HierarchicalZipWriter(protection.release(), commands_.IsZip64()));
        zip_->SetAllowUtf8(allowUtf8_);
        isStream_ = true;
      }
      else
      {
        throw OrthancException(ErrorCode_BadSequenceOfCalls);
      }
    }

    void CancelStream()
    {
      if (zip_.get() == NULL)
      {
        throw OrthancException(ErrorCode_BadSequenceOfCalls);
      }
      else if (isStream_)
      {
        zip_->CancelStream();
      }
    }

    void Close()
    {
      if (zip_.get() == NULL)
      {
        throw OrthancException(ErrorCode_BadSequenceOfCalls);
      }
      else
      {
        zip_->Close();
      }
    }

    uint64_t GetArchiveSize() const
    {
      if (zip_.get() == NULL)
      {
        throw OrthancException(ErrorCode_BadSequenceOfCalls);
      }
      else
      {
        return zip_->GetArchiveSize();
      }
    }

    size_t GetStepsCount() const
    {
      return commands_.GetSize() + 1;
    }

    void PreloadAllCommands(DicomSequentialReader& instancesLoader)
    {
      for (size_t i = 0; i < commands_.GetSize(); i++)
      {
        commands_.GetCommand(i).Preload(instancesLoader);
      }
    }

    void RunStep(DicomSequentialReader& instancesLoader,
                 size_t index)
    {
      if (index > commands_.GetSize())
      {
        throw OrthancException(ErrorCode_ParameterOutOfRange);
      }
      else if (zip_.get() == NULL)
      {
        throw OrthancException(ErrorCode_BadSequenceOfCalls);
      }
      else if (index == commands_.GetSize())
      {
        // Last step: Add the DICOMDIR
        if (isMedia_)
        {
          assert(dicomDir_.get() != NULL);
          std::string s;
          dicomDir_->Encode(s);

          zip_->OpenFile("DICOMDIR");
          zip_->Write(s);
        }
      }
      else
      {
        if (isMedia_)
        {
          assert(dicomDir_.get() != NULL);
          commands_.GetCommand(index).Execute(*zip_, instancesLoader, dicomDir_.get());
        }
        else
        {
          assert(dicomDir_.get() == NULL);
          commands_.GetCommand(index).Execute(*zip_, instancesLoader, NULL);
        }
      }
    }

    unsigned int GetInstancesCount() const
    {
      return commands_.GetInstancesCount();
    }

    uint64_t GetUncompressedSize() const
    {
      return commands_.GetUncompressedSize();
    }
  };


  ArchiveJob::ArchiveJob(ServerContext& context,
                         bool isMedia,
                         bool enableExtendedSopClass,
                         ResourceType jobLevel) :
    context_(context),
    archive_(new ArchiveIndex(GetArchiveResourceType(jobLevel))),  // get patient Info from this level
    isMedia_(isMedia),
    enableExtendedSopClass_(enableExtendedSopClass),
    filename_("archive.zip"),
    currentStep_(0),
    instancesCount_(0),
    uncompressedSize_(0),
    archiveSize_(0),
    transcode_(false),
    transferSyntax_(DicomTransferSyntax_LittleEndianImplicit),
    hasLossyQuality_(false),
    lossyQuality_(100),
    allowUtf8_(false)
  {
  }

  
  ArchiveJob::~ArchiveJob()
  {
    if (!mediaArchiveId_.empty())
    {
      context_.GetMediaArchive().Remove(mediaArchiveId_);
    }
  }


  void ArchiveJob::AcquireSynchronousTarget(ZipWriter::IOutputStream* target)
  {
    std::unique_ptr<ZipWriter::IOutputStream> protection(target);
    
    if (target == NULL)
    {
      throw OrthancException(ErrorCode_NullPointer);
    }
    else if (writer_.get() != NULL ||  // Already started
             synchronousTarget_.get() != NULL ||
             asynchronousTarget_.get() != NULL)
    {
      throw OrthancException(ErrorCode_BadSequenceOfCalls);
    }
    else
    {
      synchronousTarget_.reset(protection.release());
    }
  }


  void ArchiveJob::SetDescription(const std::string& description)
  {
    if (writer_.get() != NULL)   // Already started
    {
      throw OrthancException(ErrorCode_BadSequenceOfCalls);
    }
    else
    {
      description_ = description;
    }
  }

  void ArchiveJob::SetFilename(const std::string& filename)
  {
    if (writer_.get() != NULL)   // Already started
    {
      throw OrthancException(ErrorCode_BadSequenceOfCalls);
    }
    else
    {
      filename_ = filename;
    }
  }

  void ArchiveJob::AddResource(const std::string& publicId,
                               bool mustExist,
                               ResourceType expectedType)
  {
    if (writer_.get() != NULL)   // Already started
    {
      throw OrthancException(ErrorCode_BadSequenceOfCalls);
    }
    else
    {
      if (mustExist)
      {
        ResourceType type;
        if (!context_.GetIndex().LookupResourceType(type, publicId) ||
            type != expectedType)
        {
          throw OrthancException(ErrorCode_InexistentItem,
                                 "Missing resource while creating an archive: " + publicId);
        }
      }
      
      ResourceIdentifiers resource(context_.GetIndex(), publicId);
      archive_->Add(context_.GetIndex(), resource);
    }
  }


  void ArchiveJob::SetTranscode(DicomTransferSyntax transferSyntax)
  {
    if (writer_.get() != NULL)   // Already started
    {
      throw OrthancException(ErrorCode_BadSequenceOfCalls);
    }
    else
    {
      transcode_ = true;
      transferSyntax_ = transferSyntax;
    }
  }


  void ArchiveJob::SetLossyQuality(unsigned int lossyQuality)
  {
    if (writer_.get() != NULL)   // Already started
    {
      throw OrthancException(ErrorCode_BadSequenceOfCalls);
    }
    else
    {
      hasLossyQuality_ = true;
      lossyQuality_ = lossyQuality;
    }
  }


  void ArchiveJob::SetAllowUtf8(bool allowUtf8)
  {
    if (writer_.get() != NULL)   // Already started
    {
      throw OrthancException(ErrorCode_BadSequenceOfCalls);
    }
    else
    {
      allowUtf8_ = allowUtf8;
    }
  }


  void ArchiveJob::Reset()
  {
    throw OrthancException(ErrorCode_BadSequenceOfCalls,
                           "Cannot resubmit the creation of an archive");
  }

  
  void ArchiveJob::Start()
  {
    if (writer_.get() != NULL)
    {
      throw OrthancException(ErrorCode_BadSequenceOfCalls);
    }
    else
    {
      if (synchronousTarget_.get() == NULL)
      {
        if (asynchronousTarget_.get() != NULL)
        {
          // It is up to this method to create the asynchronous target
          THROW_WITH_FILE_AND_LINE_INFO(ErrorCode_InternalError);
        }
        else
        {
          OrthancConfiguration::ReaderLock lock;
          asynchronousTarget_.reset(lock.GetConfiguration().CreateTemporaryFile());
          
          assert(asynchronousTarget_.get() != NULL);
          asynchronousTarget_->Touch();  // Make sure we can write to the temporary file
          
          writer_.reset(new ZipWriterIterator(context_, *archive_, isMedia_, enableExtendedSopClass_, allowUtf8_));
          writer_->SetOutputFile(asynchronousTarget_->GetPath());
        }
      }
      else
      {
        assert(synchronousTarget_.get() != NULL);
    
        writer_.reset(new ZipWriterIterator(context_, *archive_, isMedia_, enableExtendedSopClass_, allowUtf8_));
        writer_->AcquireOutputStream(synchronousTarget_.release());
      }

      instancesCount_ = writer_->GetInstancesCount();
      uncompressedSize_ = writer_->GetUncompressedSize();

      if (transcode_)
      {
        instancesLoader_.reset(context_.GetDicomSequentialReaderFactory().CreateForTranscodedRawMemoryBuffer(
                                 transferSyntax_, TranscodingSopInstanceUidMode_AllowNew, hasLossyQuality_, lossyQuality_));
      }
      else
      {
        instancesLoader_.reset(context_.GetDicomSequentialReaderFactory().CreateForOriginalRawMemoryBuffer());
      }

      writer_->PreloadAllCommands(*instancesLoader_);

      instancesLoader_->Start();
    }
  }



  namespace
  {
    class DynamicTemporaryFile : public IDynamicObject
    {
    private:
      std::unique_ptr<TemporaryFile>   file_;

    public:
      explicit DynamicTemporaryFile(TemporaryFile* f) : file_(f)
      {
        if (f == NULL)
        {
          throw OrthancException(ErrorCode_NullPointer);
        }
      }

      const TemporaryFile& GetFile() const
      {
        assert(file_.get() != NULL);
        return *file_;
      }
    };
  }
  

  void ArchiveJob::FinalizeTarget()
  {
    if (writer_.get() != NULL)
    {
      writer_->Close();  // Flush all the results
      archiveSize_ = writer_->GetArchiveSize();
      writer_.reset();
    }

    instancesLoader_.reset(NULL);

    if (asynchronousTarget_.get() != NULL)
    {
      // Asynchronous behavior: Move the resulting file into the media archive
      mediaArchiveId_ = context_.GetMediaArchive().Add(
        new DynamicTemporaryFile(asynchronousTarget_.release()));
    }
  }
    

  JobStepResult ArchiveJob::Step(const std::string& jobId)
  {
    assert(writer_.get() != NULL);

    if (writer_->GetStepsCount() == 0)
    {
      FinalizeTarget();
      return JobStepResult::Success();
    }
    else
    {
      try
      {
        writer_->RunStep(*instancesLoader_, currentStep_);
      }
      catch (Orthanc::OrthancException& e)
      {
        LOG(ERROR) << "Error while creating an archive: " << e.What();
        writer_->CancelStream();
        throw;
      }

      currentStep_ ++;

      if (currentStep_ == writer_->GetStepsCount())
      {
        FinalizeTarget();
        return JobStepResult::Success();
      }
      else
      {
        archiveSize_ = writer_->GetArchiveSize();
        return JobStepResult::Continue();
      }
    }
  }


  void ArchiveJob::Stop(JobStopReason reason)
  {
    /**
     * New in Orthanc 1.9.3: Remove the temporary file associated with
     * the job as soon as its job gets canceled (especially visible in
     * asynchronous mode).
     **/
    if (reason == JobStopReason_Canceled ||
        reason == JobStopReason_Failure ||
        reason == JobStopReason_Retry)
    {
      writer_->CancelStream();
      
      // First delete the writer, as it holds a reference to "(a)synchronousTarget_", cf. (*)
      writer_.reset();
      
      synchronousTarget_.reset();
      asynchronousTarget_.reset();

      // clear the loader threads
      instancesLoader_.reset(NULL);
    }
  }


  float ArchiveJob::GetProgress() const
  {
    if (writer_.get() == NULL ||
        writer_->GetStepsCount() == 0)
    {
      return 1;
    }
    else
    {
      return (static_cast<float>(currentStep_) /
              static_cast<float>(writer_->GetStepsCount() - 1));
    }
  }

    
  void ArchiveJob::GetJobType(std::string& target) const
  {
    if (isMedia_)
    {
      target = "Media";
    }
    else
    {
      target = "Archive";
    }
  }


  void ArchiveJob::GetPublicContent(Json::Value& value) const
  {
    value = Json::objectValue;
    value[KEY_DESCRIPTION] = description_;
    value[KEY_INSTANCES_COUNT] = instancesCount_;
    value[KEY_UNCOMPRESSED_SIZE_MB] = static_cast<unsigned int>(uncompressedSize_ / MEGABYTE);
    value[KEY_ARCHIVE_SIZE_MB] = static_cast<unsigned int>(archiveSize_ / MEGABYTE);

    // New in Orthanc 1.9.4
    value[KEY_ARCHIVE_SIZE] = boost::lexical_cast<std::string>(archiveSize_);
    value[KEY_UNCOMPRESSED_SIZE] = boost::lexical_cast<std::string>(uncompressedSize_);

    if (transcode_)
    {
      value[KEY_TRANSCODE] = GetTransferSyntaxUid(transferSyntax_);
    }

    // New in Orthanc 1.12.11
    value[KEY_ALLOW_UTF8] = allowUtf8_;

    // New in Orthanc 1.13.0
    if (hasLossyQuality_)
    {
      value[KEY_LOSSY_QUALITY] = lossyQuality_;
    }
  }


  bool ArchiveJob::GetOutput(std::string& output,
                             MimeType& mime,
                             std::string& filename,
                             const std::string& key)
  {   
    if (key == "archive" &&
        !mediaArchiveId_.empty())
    {
      SharedArchive::Accessor accessor(context_.GetMediaArchive(), mediaArchiveId_);

      if (accessor.IsValid())
      {
        const DynamicTemporaryFile& f = dynamic_cast<DynamicTemporaryFile&>(accessor.GetItem());
        f.GetFile().Read(output);
        mime = MimeType_Zip;
        filename = filename_;
        return true;
      }
      else
      {
        return false;
      }
    }    
    else
    {
      return false;
    }
  }

  bool ArchiveJob::DeleteOutput(const std::string& key)
  {   
    if (key == "archive" &&
        !mediaArchiveId_.empty())
    {
      bool found;

      {
        SharedArchive::Accessor accessor(context_.GetMediaArchive(), mediaArchiveId_);
        found = accessor.IsValid();
      }

      context_.GetMediaArchive().Remove(mediaArchiveId_);
      return found;
    }
    else
    {
      return false;
    }
  }

  void ArchiveJob::DeleteAllOutputs()
  {
    DeleteOutput("archive");
  }
}
