1 files changed, 432 insertions, 0 deletions

diff --git a/xbmc/utils/ScraperUrl.cpp b/xbmc/utils/ScraperUrl.cpp
new file mode 100644
index 0000000..f242a40
--- /dev/null
+++ b/xbmc/utils/ScraperUrl.cpp
@@ -0,0 +1,432 @@
+/*
+ *  Copyright (C) 2005-2018 Team Kodi
+ *  This file is part of Kodi - https://kodi.tv
+ *
+ *  SPDX-License-Identifier: GPL-2.0-or-later
+ *  See LICENSES/README.md for more information.
+ */
+
+#include "ScraperUrl.h"
+
+#include "CharsetConverter.h"
+#include "ServiceBroker.h"
+#include "URIUtils.h"
+#include "URL.h"
+#include "XMLUtils.h"
+#include "filesystem/CurlFile.h"
+#include "filesystem/ZipFile.h"
+#include "settings/AdvancedSettings.h"
+#include "settings/SettingsComponent.h"
+#include "utils/CharsetDetection.h"
+#include "utils/Mime.h"
+#include "utils/StringUtils.h"
+#include "utils/XBMCTinyXML.h"
+#include "utils/log.h"
+
+#include <algorithm>
+#include <cstring>
+#include <sstream>
+
+CScraperUrl::CScraperUrl() : m_relevance(0.0), m_parsed(false)
+{
+}
+
+CScraperUrl::CScraperUrl(std::string strUrl) : CScraperUrl()
+{
+  ParseFromData(std::move(strUrl));
+}
+
+CScraperUrl::CScraperUrl(const TiXmlElement* element) : CScraperUrl()
+{
+  ParseAndAppendUrl(element);
+}
+
+CScraperUrl::~CScraperUrl() = default;
+
+void CScraperUrl::Clear()
+{
+  m_urls.clear();
+  m_data.clear();
+  m_relevance = 0.0;
+  m_parsed = false;
+}
+
+void CScraperUrl::SetData(std::string data)
+{
+  m_data = std::move(data);
+  m_parsed = false;
+}
+
+const CScraperUrl::SUrlEntry CScraperUrl::GetFirstUrlByType(const std::string& type) const
+{
+  const auto url = std::find_if(m_urls.begin(), m_urls.end(), [type](const SUrlEntry& url) {
+    return url.m_type == UrlType::General && (type.empty() || url.m_aspect == type);
+  });
+  if (url != m_urls.end())
+    return *url;
+
+  return SUrlEntry();
+}
+
+const CScraperUrl::SUrlEntry CScraperUrl::GetSeasonUrl(int season, const std::string& type) const
+{
+  const auto url = std::find_if(m_urls.begin(), m_urls.end(), [season, type](const SUrlEntry& url) {
+    return url.m_type == UrlType::Season && url.m_season == season &&
+           (type.empty() || type == "thumb" || url.m_aspect == type);
+  });
+  if (url != m_urls.end())
+    return *url;
+
+  return SUrlEntry();
+}
+
+unsigned int CScraperUrl::GetMaxSeasonUrl() const
+{
+  unsigned int maxSeason = 0;
+  for (const auto& url : m_urls)
+  {
+    if (url.m_type == UrlType::Season && url.m_season > 0 &&
+        static_cast<unsigned int>(url.m_season) > maxSeason)
+      maxSeason = url.m_season;
+  }
+  return maxSeason;
+}
+
+std::string CScraperUrl::GetFirstThumbUrl() const
+{
+  if (m_urls.empty())
+    return {};
+
+  return GetThumbUrl(m_urls.front());
+}
+
+void CScraperUrl::GetThumbUrls(std::vector<std::string>& thumbs,
+                               const std::string& type,
+                               int season,
+                               bool unique) const
+{
+  for (const auto& url : m_urls)
+  {
+    if (url.m_aspect == type || type.empty() || url.m_aspect.empty())
+    {
+      if ((url.m_type == CScraperUrl::UrlType::General && season == -1) ||
+          (url.m_type == CScraperUrl::UrlType::Season && url.m_season == season))
+      {
+        std::string thumbUrl = GetThumbUrl(url);
+        if (!unique || std::find(thumbs.begin(), thumbs.end(), thumbUrl) == thumbs.end())
+          thumbs.push_back(thumbUrl);
+      }
+    }
+  }
+}
+
+bool CScraperUrl::Parse()
+{
+  if (m_parsed)
+    return true;
+
+  auto dataToParse = m_data;
+  m_data.clear();
+  return ParseFromData(std::move(dataToParse));
+}
+
+bool CScraperUrl::ParseFromData(std::string data)
+{
+  if (data.empty())
+    return false;
+
+  CXBMCTinyXML doc;
+  /* strUrl is coming from internal sources (usually generated by scraper or from database)
+   * so strUrl is always in UTF-8 */
+  doc.Parse(data, TIXML_ENCODING_UTF8);
+
+  auto pElement = doc.RootElement();
+  if (pElement == nullptr)
+  {
+    m_urls.emplace_back(data);
+    m_data = data;
+  }
+  else
+  {
+    while (pElement != nullptr)
+    {
+      ParseAndAppendUrl(pElement);
+      pElement = pElement->NextSiblingElement(pElement->Value());
+    }
+  }
+
+  m_parsed = true;
+  return true;
+}
+
+bool CScraperUrl::ParseAndAppendUrl(const TiXmlElement* element)
+{
+  if (element == nullptr || element->FirstChild() == nullptr ||
+      element->FirstChild()->Value() == nullptr)
+    return false;
+
+  bool wasEmpty = m_data.empty();
+
+  std::stringstream stream;
+  stream << *element;
+  m_data += stream.str();
+
+  SUrlEntry url(element->FirstChild()->ValueStr());
+  url.m_spoof = XMLUtils::GetAttribute(element, "spoof");
+
+  const char* szPost = element->Attribute("post");
+  if (szPost && StringUtils::CompareNoCase(szPost, "yes") == 0)
+    url.m_post = true;
+  else
+    url.m_post = false;
+
+  const char* szIsGz = element->Attribute("gzip");
+  if (szIsGz && StringUtils::CompareNoCase(szIsGz, "yes") == 0)
+    url.m_isgz = true;
+  else
+    url.m_isgz = false;
+
+  url.m_cache = XMLUtils::GetAttribute(element, "cache");
+
+  const char* szType = element->Attribute("type");
+  if (szType && StringUtils::CompareNoCase(szType, "season") == 0)
+  {
+    url.m_type = UrlType::Season;
+    const char* szSeason = element->Attribute("season");
+    if (szSeason)
+      url.m_season = atoi(szSeason);
+  }
+
+  url.m_aspect = XMLUtils::GetAttribute(element, "aspect");
+
+  m_urls.push_back(url);
+
+  if (wasEmpty)
+    m_parsed = true;
+
+  return true;
+}
+
+// XML format is of strUrls is:
+// <TAG><url>...</url>...</TAG> (parsed by ParseElement) or <url>...</url> (ditto)
+bool CScraperUrl::ParseAndAppendUrlsFromEpisodeGuide(std::string episodeGuide)
+{
+  if (episodeGuide.empty())
+    return false;
+
+  // ok, now parse the xml file
+  CXBMCTinyXML doc;
+  /* strUrls is coming from internal sources so strUrls is always in UTF-8 */
+  doc.Parse(episodeGuide, TIXML_ENCODING_UTF8);
+  if (doc.RootElement() == nullptr)
+    return false;
+
+  bool wasEmpty = m_data.empty();
+
+  TiXmlHandle docHandle(&doc);
+  auto link = docHandle.FirstChild("episodeguide").Element();
+  if (link->FirstChildElement("url"))
+  {
+    for (link = link->FirstChildElement("url"); link; link = link->NextSiblingElement("url"))
+      ParseAndAppendUrl(link);
+  }
+  else if (link->FirstChild() && link->FirstChild()->Value())
+    ParseAndAppendUrl(link);
+
+  if (wasEmpty)
+    m_parsed = true;
+
+  return true;
+}
+
+void CScraperUrl::AddParsedUrl(std::string url,
+                               std::string aspect,
+                               std::string preview,
+                               std::string referrer,
+                               std::string cache,
+                               bool post,
+                               bool isgz,
+                               int season)
+{
+  bool wasEmpty = m_data.empty();
+
+  TiXmlElement thumb("thumb");
+  thumb.SetAttribute("spoof", referrer);
+  thumb.SetAttribute("cache", cache);
+  if (post)
+    thumb.SetAttribute("post", "yes");
+  if (isgz)
+    thumb.SetAttribute("gzip", "yes");
+  if (season >= 0)
+  {
+    thumb.SetAttribute("season", StringUtils::Format("%i", season));
+    thumb.SetAttribute("type", "season");
+  }
+  thumb.SetAttribute("aspect", aspect);
+  thumb.SetAttribute("preview", preview);
+  TiXmlText text(url);
+  thumb.InsertEndChild(text);
+
+  m_data << thumb;
+
+  SUrlEntry nUrl(url);
+  nUrl.m_spoof = referrer;
+  nUrl.m_post = post;
+  nUrl.m_isgz = isgz;
+  nUrl.m_cache = cache;
+  if (season >= 0)
+  {
+    nUrl.m_type = UrlType::Season;
+    nUrl.m_season = season;
+  }
+  nUrl.m_aspect = aspect;
+
+  m_urls.push_back(nUrl);
+
+  if (wasEmpty)
+    m_parsed = true;
+}
+
+std::string CScraperUrl::GetThumbUrl(const CScraperUrl::SUrlEntry& entry)
+{
+  if (entry.m_spoof.empty())
+    return entry.m_url;
+
+  return entry.m_url + "|Referer=" + CURL::Encode(entry.m_spoof);
+}
+
+bool CScraperUrl::Get(const SUrlEntry& scrURL,
+                      std::string& strHTML,
+                      XFILE::CCurlFile& http,
+                      const std::string& cacheContext)
+{
+  CURL url(scrURL.m_url);
+  http.SetReferer(scrURL.m_spoof);
+  std::string strCachePath;
+
+  if (!scrURL.m_cache.empty())
+  {
+    strCachePath = URIUtils::AddFileToFolder(
+        CServiceBroker::GetSettingsComponent()->GetAdvancedSettings()->m_cachePath, "scrapers",
+        cacheContext, scrURL.m_cache);
+    if (XFILE::CFile::Exists(strCachePath))
+    {
+      XFILE::CFile file;
+      XFILE::auto_buffer buffer;
+      if (file.LoadFile(strCachePath, buffer) > 0)
+      {
+        strHTML.assign(buffer.get(), buffer.length());
+        return true;
+      }
+    }
+  }
+
+  auto strHTML1 = strHTML;
+
+  if (scrURL.m_post)
+  {
+    std::string strOptions = url.GetOptions();
+    strOptions = strOptions.substr(1);
+    url.SetOptions("");
+
+    if (!http.Post(url.Get(), strOptions, strHTML1))
+      return false;
+  }
+  else if (!http.Get(url.Get(), strHTML1))
+    return false;
+
+  strHTML = strHTML1;
+
+  const auto mimeType = http.GetProperty(XFILE::FILE_PROPERTY_MIME_TYPE);
+  CMime::EFileType ftype = CMime::GetFileTypeFromMime(mimeType);
+  if (ftype == CMime::FileTypeUnknown)
+    ftype = CMime::GetFileTypeFromContent(strHTML);
+
+  if (ftype == CMime::FileTypeZip || ftype == CMime::FileTypeGZip)
+  {
+    XFILE::CZipFile file;
+    std::string strBuffer;
+    auto iSize = file.UnpackFromMemory(
+        strBuffer, strHTML, scrURL.m_isgz); // FIXME: use FileTypeGZip instead of scrURL.m_isgz?
+    if (iSize > 0)
+    {
+      strHTML = strBuffer;
+      CLog::Log(LOGDEBUG, "{}: Archive \"{}\" was unpacked in memory", __FUNCTION__, scrURL.m_url);
+    }
+    else
+      CLog::Log(LOGWARNING, "{}: \"{}\" looks like archive but cannot be unpacked", __FUNCTION__,
+                scrURL.m_url);
+  }
+
+  const auto reportedCharset = http.GetProperty(XFILE::FILE_PROPERTY_CONTENT_CHARSET);
+  if (ftype == CMime::FileTypeHtml)
+  {
+    std::string realHtmlCharset, converted;
+    if (!CCharsetDetection::ConvertHtmlToUtf8(strHTML, converted, reportedCharset, realHtmlCharset))
+      CLog::Log(LOGWARNING,
+                "{}: Can't find precise charset for HTML \"{}\", using \"{}\" as fallback",
+                __FUNCTION__, scrURL.m_url, realHtmlCharset);
+    else
+      CLog::Log(LOGDEBUG, "{}: Using \"{}\" charset for HTML \"{}\"", __FUNCTION__, realHtmlCharset,
+                scrURL.m_url);
+
+    strHTML = converted;
+  }
+  else if (ftype == CMime::FileTypeXml)
+  {
+    CXBMCTinyXML xmlDoc;
+    xmlDoc.Parse(strHTML, reportedCharset);
+
+    const auto realXmlCharset = xmlDoc.GetUsedCharset();
+    if (!realXmlCharset.empty())
+    {
+      CLog::Log(LOGDEBUG, "{}: Using \"{}\" charset for XML \"{}\"", __FUNCTION__, realXmlCharset,
+                scrURL.m_url);
+      std::string converted;
+      g_charsetConverter.ToUtf8(realXmlCharset, strHTML, converted);
+      strHTML = converted;
+    }
+  }
+  else if (ftype == CMime::FileTypePlainText ||
+           StringUtils::EqualsNoCase(mimeType.substr(0, 5), "text/"))
+  {
+    std::string realTextCharset;
+    std::string converted;
+    CCharsetDetection::ConvertPlainTextToUtf8(strHTML, converted, reportedCharset, realTextCharset);
+    strHTML = converted;
+    if (reportedCharset != realTextCharset)
+      CLog::Log(LOGWARNING,
+                "{}: Using \"{}\" charset for plain text \"{}\" instead of server reported \"{}\" "
+                "charset",
+                __FUNCTION__, realTextCharset, scrURL.m_url, reportedCharset);
+    else
+      CLog::Log(LOGDEBUG, "{}: Using \"{}\" charset for plain text \"{}\"", __FUNCTION__,
+                realTextCharset, scrURL.m_url);
+  }
+  else if (!reportedCharset.empty())
+  {
+    CLog::Log(LOGDEBUG, "{}: Using \"{}\" charset for \"{}\"", __FUNCTION__, reportedCharset,
+              scrURL.m_url);
+    if (reportedCharset != "UTF-8")
+    {
+      std::string converted;
+      g_charsetConverter.ToUtf8(reportedCharset, strHTML, converted);
+      strHTML = converted;
+    }
+  }
+  else
+    CLog::Log(LOGDEBUG, "{}: Using content of \"{}\" as binary or text with \"UTF-8\" charset",
+              __FUNCTION__, scrURL.m_url);
+
+  if (!scrURL.m_cache.empty())
+  {
+    const auto strCachePath = URIUtils::AddFileToFolder(
+        CServiceBroker::GetSettingsComponent()->GetAdvancedSettings()->m_cachePath, "scrapers",
+        cacheContext, scrURL.m_cache);
+    XFILE::CFile file;
+    if (!file.OpenForWrite(strCachePath, true) ||
+        file.Write(strHTML.data(), strHTML.size()) != static_cast<ssize_t>(strHTML.size()))
+      return false;
+  }
+  return true;
+}