add pvgis to iotools (pvlib#845)

* add get_pvgis_tmy to iotools

* stickler fixes maybe?

* add pvgis tests

- upload test data
- check the JSON output format
- add get_pvgis_tmy to api
- add timeout and url args as suggested
- add raises to explain error messages
- correct user horizon docstring decription

* fix stickler

* STY: fix indents and long lines in test meta JSON

* check each column of output separately

* add test for basic output format

* check error raised if bad output format
* must use NamedTemporaryFile with delete=True
* add test for csv output format

* add test for epw output format

* add parse_epw to use buffer v. filepath

* add test data for pvgis tmy userhorizon
* add parse_epw to iotools api
* add tests for usehorizon & userhorizon
* check meta from epw
* in test_epw, meta not used so replace with _
* pvgis-tmy-epw don't use temp files anymore

* fix use lowercase url in request

* test startyear and endyear
* test a bad url, returns 404

* use assert cond is False instead of ==, thx stickler

* update api docs and what's new

* fix typo in Parameters

* use with context manager for IO

* instead of try:finally
* fix defaults notation
* add reference links and improve docstring,

* use fixtures for test_pvgis

* also move META dict to a json file in pvlib/data

* fix reference links

* fix open for py35

* also add references header in pvgis.py

* remove whitespace on blank line

* initialize data to None in case API fails to respond to bad outputformat

Author: mikofski

docs/sphinx/source/api.rst

@@ -359,6 +359,7 @@ relevant to solar energy modeling.
    iotools.read_tmy2
    iotools.read_tmy3
    iotools.read_epw
+   iotools.parse_epw
    iotools.read_srml
    iotools.read_srml_month_from_solardat
    iotools.read_surfrad
@@ -371,6 +372,7 @@ relevant to solar energy modeling.
    iotools.get_psm3
    iotools.read_psm3
    iotools.parse_psm3
+   iotools.get_pvgis_tmy
 
 A :py:class:`~pvlib.location.Location` object may be created from metadata
 in some files.

docs/sphinx/source/whatsnew/v0.7.1.rst

@@ -14,7 +14,9 @@ Enhancements
   objects. (:issue:`841`)
 * Added `leap_day` parameter to `iotools.get_psm3` instead of hardcoding it as
   False.
-
+* Added :py:func:`~pvlib.iotools.get_pvgis_tmy` to get PVGIS TMY datasets.
+* Added :py:func:`~pvlib.iotools.parse_epw` to parse a file-like buffer
+  containing weather data in the EPW format.
 
 Bug fixes
 ~~~~~~~~~
@@ -33,3 +35,4 @@ Documentation
 Contributors
 ~~~~~~~~~~~~
 * Kevin Anderson (:ghuser:`kanderso-nrel`)
+* Mark Mikofski (:ghuser:`mikofski`)

pvlib/data/pvgis_tmy_meta.json

@@ -0,0 +1,93 @@
+{
+  "inputs": {
+    "location": {
+      "description": "Selected location",
+      "variables": {
+        "latitude": {
+          "description": "Latitude",
+          "units": "decimal degree"
+        },
+        "longitude": {
+          "description": "Longitude",
+          "units": "decimal degree"
+        },
+        "elevation": {
+          "description": "Elevation",
+          "units": "m"
+        }
+      }
+    },
+    "meteo_data": {
+      "description": "Sources of meteorological data",
+      "variables": {
+        "radiation_db": {
+          "description": "Solar radiation database"
+        },
+        "meteo_db": {
+          "description": "Database used for meteorological variables other than solar radiation"
+        },
+        "year_min": {
+          "description": "First year of the calculations"
+        },
+        "year_max": {
+          "description": "Last year of the calculations"
+        },
+        "use_horizon": {
+          "description": "Include horizon shadows"
+        },
+        "horizon_db": {
+          "description": "Source of horizon data"
+        }
+      }
+    }
+  },
+  "outputs": {
+    "months_selected": {
+      "type": "time series",
+      "timestamp": "monthly",
+      "description": "months selected for the TMY"
+    },
+    "tmy_hourly": {
+      "type": "time series",
+      "timestamp": "hourly",
+      "variables": {
+        "T2m": {
+          "description": "2-m air temperature",
+          "units": "degree Celsius"
+        },
+        "RH": {
+          "description": "relative humidity",
+          "units": "%"
+        },
+        "G(h)": {
+          "description": "Global irradiance on the horizontal plane",
+          "units": "W/m2"
+        },
+        "Gb(n)": {
+          "description": "Beam/direct irradiance on a plane always normal to sun rays",
+          "units": "W/m2"
+        },
+        "Gd(h)": {
+          "description": "Diffuse irradiance on the horizontal plane",
+          "units": "W/m2"
+        },
+        "IR(h)": {
+          "description": "Surface infrared (thermal) irradiance on a horizontal plane",
+          "units": "W/m2"
+        },
+        "WS10m": {
+          "description": "10-m total wind speed",
+          "units": "m/s"
+        },
+        "WD10m": {
+          "description": "10-m wind direction (0 = N, 90 = E)",
+          "units": "degree"
+        },
+        "SP": {
+          "description": "Surface (air) pressure",
+          "units": "Pa"
+        }
+      }
+    }
+  }
+}

pvlib/data/pvgis_tmy_test.dat

@@ -1,6 +1,6 @@
 from pvlib.iotools.tmy import read_tmy2  # noqa: F401
 from pvlib.iotools.tmy import read_tmy3  # noqa: F401
-from pvlib.iotools.epw import read_epw  # noqa: F401
+from pvlib.iotools.epw import read_epw, parse_epw  # noqa: F401
 from pvlib.iotools.srml import read_srml  # noqa: F401
 from pvlib.iotools.srml import read_srml_month_from_solardat  # noqa: F401
 from pvlib.iotools.surfrad import read_surfrad  # noqa: F401
@@ -13,3 +13,4 @@
 from pvlib.iotools.psm3 import get_psm3  # noqa: F401
 from pvlib.iotools.psm3 import read_psm3  # noqa: F401
 from pvlib.iotools.psm3 import parse_psm3  # noqa: F401
+from pvlib.iotools.pvgis import get_pvgis_tmy  # noqa: F401

pvlib/data/tmy_45.000_8.000_userhorizon.json

@@ -35,17 +35,18 @@ def read_epw(filename, coerce_year=None):
 
     Returns
     -------
-    Tuple of the form (data, metadata).
-
     data : DataFrame
         A pandas dataframe with the columns described in the table
         below. For more detailed descriptions of each component, please
-        consult the EnergyPlus Auxiliary Programs documentation
-        available at: https://energyplus.net/documentation.
+        consult the EnergyPlus Auxiliary Programs documentation [1]_
 
     metadata : dict
         The site metadata available in the file.
 
+    See Also
+    --------
+    pvlib.iotools.parse_epw
+
     Notes
     -----
 
@@ -111,8 +112,8 @@ def read_epw(filename, coerce_year=None):
     References
     ----------
 
-    .. [1] EnergyPlus documentation, Auxiliary Programs
-       https://energyplus.net/documentation.
+    .. [1] `EnergyPlus documentation, Auxiliary Programs
+       <https://energyplus.net/documentation>`_
     '''
 
     if filename.startswith('http'):
@@ -127,7 +128,45 @@ def read_epw(filename, coerce_year=None):
     else:
         # Assume it's accessible via the file system
         csvdata = open(filename, 'r')
+    try:
+        data, meta = parse_epw(csvdata, coerce_year)
+    finally:
+        csvdata.close()
+    return data, meta
+
+
+def parse_epw(csvdata, coerce_year=None):
+    """
+    Given a file-like buffer with data in Energy Plus Weather (EPW) format,
+    parse the data into a dataframe.
+
+    Parameters
+    ----------
+    csvdata : file-like buffer
+        a file-like buffer containing data in the EPW format
+
+    coerce_year : None or int, default None
+        If supplied, the year of the data will be set to this value. This can
+        be a useful feature because EPW data is composed of data from
+        different years.
+        Warning: EPW files always have 365*24 = 8760 data rows;
+        be careful with the use of leap years.
+
+    Returns
+    -------
+    data : DataFrame
+        A pandas dataframe with the columns described in the table
+        below. For more detailed descriptions of each component, please
+        consult the EnergyPlus Auxiliary Programs documentation
+        available at: https://energyplus.net/documentation.
+
+    metadata : dict
+        The site metadata available in the file.
 
+    See Also
+    --------
+    pvlib.iotools.read_epw
+    """
     # Read line with metadata
     firstline = csvdata.readline()
 

pvlib/iotools/__init__.py

@@ -0,0 +1,176 @@
+"""
+Get, read, and parse data from `PVGIS <https://ec.europa.eu/jrc/en/pvgis>`_.
+
+For more information, see the following links:
+* `Interactive Tools <https://re.jrc.ec.europa.eu/pvg_tools/en/tools.html>`_
+* `Data downloads <https://ec.europa.eu/jrc/en/PVGIS/downloads/data>`_
+* `User manual docs <https://ec.europa.eu/jrc/en/PVGIS/docs/usermanual>`_
+
+More detailed information about the API for TMY and hourly radiation are here:
+* `TMY <https://ec.europa.eu/jrc/en/PVGIS/tools/tmy>`_
+* `hourly radiation
+  <https://ec.europa.eu/jrc/en/PVGIS/tools/hourly-radiation>`_
+* `daily radiation <https://ec.europa.eu/jrc/en/PVGIS/tools/daily-radiation>`_
+* `monthly radiation
+  <https://ec.europa.eu/jrc/en/PVGIS/tools/monthly-radiation>`_
+"""
+import io
+import requests
+import pandas as pd
+from pvlib.iotools import parse_epw
+
+URL = 'https://re.jrc.ec.europa.eu/api/'
+
+
+def get_pvgis_tmy(lat, lon, outputformat='json', usehorizon=True,
+                  userhorizon=None, startyear=None, endyear=None, url=URL,
+                  timeout=30):
+    """
+    Get TMY data from PVGIS [1]_. For more information see the PVGIS TMY tool
+    documentation [2]_.
+
+    Parameters
+    ----------
+    lat : float
+        Latitude in degrees north
+    lon : float
+        Longitude in dgrees east
+    outputformat : str, default 'json'
+        Must be in ``['csv', 'basic', 'epw', 'json']``. See PVGIS TMY tool
+        documentation [2]_ for more info.
+    usehorizon : bool, default True
+        include effects of horizon
+    userhorizon : list of float, default None
+        optional user specified elevation of horizon in degrees, at equally
+        spaced azimuth clockwise from north, only valid if `usehorizon` is
+        true, if `usehorizon` is true but `userhorizon` is `None` then PVGIS
+        will calculate the horizon [3]_
+    startyear : int, default None
+        first year to calculate TMY
+    endyear : int, default None
+        last year to calculate TMY, must be at least 10 years from first year
+    url : str, default :const:`pvlib.iotools.pvgis.URL`
+        base url of PVGIS API, append ``tmy`` to get TMY endpoint
+    timeout : int, default 30
+        time in seconds to wait for server response before timeout
+
+    Returns
+    -------
+    data : pandas.DataFrame
+        the weather data
+    months_selected : list
+        TMY year for each month, ``None`` for basic and EPW
+    inputs : dict
+        the inputs, ``None`` for basic and EPW
+    meta : list or dict
+        meta data, ``None`` for basic
+
+    Raises
+    ------
+    requests.HTTPError
+        if the request response status is ``HTTP/1.1 400 BAD REQUEST``, then
+        the error message in the response will be raised as an exception,
+        otherwise raise whatever ``HTTP/1.1`` error occurred
+
+    References
+    ----------
+
+    .. [1] `PVGIS <https://ec.europa.eu/jrc/en/pvgis>`_
+    .. [2] `PVGIS TMY tool <https://ec.europa.eu/jrc/en/PVGIS/tools/tmy>`_
+    .. [3] `PVGIS horizon profile tool
+       <https://ec.europa.eu/jrc/en/PVGIS/tools/horizon>`_
+    """
+    # use requests to format the query string by passing params dictionary
+    params = {'lat': lat, 'lon': lon, 'outputformat': outputformat}
+    # pvgis only likes 0 for False, and 1 for True, not strings, also the
+    # default for usehorizon is already 1 (ie: True), so only set if False
+    if not usehorizon:
+        params['usehorizon'] = 0
+    if userhorizon is not None:
+        params['userhorizon'] = ','.join(str(x) for x in userhorizon)
+    if startyear is not None:
+        params['startyear'] = startyear
+    if endyear is not None:
+        params['endyear'] = endyear
+    res = requests.get(url + 'tmy', params=params, timeout=timeout)
+    # PVGIS returns really well formatted error messages in JSON for HTTP/1.1
+    # 400 BAD REQUEST so try to return that if possible, otherwise raise the
+    # HTTP/1.1 error caught by requests
+    if not res.ok:
+        try:
+            err_msg = res.json()
+        except Exception:
+            res.raise_for_status()
+        else:
+            raise requests.HTTPError(err_msg['message'])
+    # initialize data to None in case API fails to respond to bad outputformat
+    data = None, None, None, None
+    if outputformat == 'json':
+        src = res.json()
+        return _parse_pvgis_tmy_json(src)
+    elif outputformat == 'csv':
+        with io.BytesIO(res.content) as src:
+            data = _parse_pvgis_tmy_csv(src)
+    elif outputformat == 'basic':
+        with io.BytesIO(res.content) as src:
+            data = _parse_pvgis_tmy_basic(src)
+    elif outputformat == 'epw':
+        with io.StringIO(res.content.decode('utf-8')) as src:
+            data, meta = parse_epw(src)
+            data = (data, None, None, meta)
+    else:
+        # this line is never reached because if outputformat is not valid then
+        # the response is HTTP/1.1 400 BAD REQUEST which is handled earlier
+        pass
+    return data
+
+
+def _parse_pvgis_tmy_json(src):
+    inputs = src['inputs']
+    meta = src['meta']
+    months_selected = src['outputs']['months_selected']
+    data = pd.DataFrame(src['outputs']['tmy_hourly'])
+    data.index = pd.to_datetime(
+        data['time(UTC)'], format='%Y%m%d:%H%M', utc=True)
+    data = data.drop('time(UTC)', axis=1)
+    return data, months_selected, inputs, meta
+
+
+def _parse_pvgis_tmy_csv(src):
+    # the first 3 rows are latitude, longitude, elevation
+    inputs = {}
+    # 'Latitude (decimal degrees): 45.000\r\n'
+    inputs['latitude'] = float(src.readline().split(b':')[1])
+    # 'Longitude (decimal degrees): 8.000\r\n'
+    inputs['longitude'] = float(src.readline().split(b':')[1])
+    # Elevation (m): 1389.0\r\n
+    inputs['elevation'] = float(src.readline().split(b':')[1])
+    # then there's a 13 row comma separated table with two columns: month, year
+    # which contains the year used for that month in the
+    src.readline()  # get "month,year\r\n"
+    months_selected = []
+    for month in range(12):
+        months_selected.append(
+            {'month': month+1, 'year': int(src.readline().split(b',')[1])})
+    # then there's the TMY (typical meteorological year) data
+    # first there's a header row:
+    #    time(UTC),T2m,RH,G(h),Gb(n),Gd(h),IR(h),WS10m,WD10m,SP
+    headers = [h.decode('utf-8').strip() for h in src.readline().split(b',')]
+    data = pd.DataFrame(
+        [src.readline().split(b',') for _ in range(8760)], columns=headers)
+    dtidx = data['time(UTC)'].apply(lambda dt: dt.decode('utf-8'))
+    dtidx = pd.to_datetime(dtidx, format='%Y%m%d:%H%M', utc=True)
+    data = data.drop('time(UTC)', axis=1)
+    data = pd.DataFrame(data, dtype=float)
+    data.index = dtidx
+    # finally there's some meta data
+    meta = [line.decode('utf-8').strip() for line in src.readlines()]
+    return data, months_selected, inputs, meta
+
+
+def _parse_pvgis_tmy_basic(src):
+    data = pd.read_csv(src)
+    data.index = pd.to_datetime(
+        data['time(UTC)'], format='%Y%m%d:%H%M', utc=True)
+    data = data.drop('time(UTC)', axis=1)
+    return data, None, None, None