diff --git a/MongoDB.ipynb b/MongoDB.ipynb index b817f97..68e9404 100644 --- a/MongoDB.ipynb +++ b/MongoDB.ipynb @@ -2,24 +2,16 @@ "cells": [ { "cell_type": "code", - "execution_count": 1, + "execution_count": 3, "metadata": {}, "outputs": [ - { - "name": "stderr", - "output_type": "stream", - "text": [ - "/usr/local/lib/python3.5/dist-packages/ipykernel_launcher.py:11: DeprecationWarning: insert is deprecated. Use insert_one or insert_many instead.\n", - " # This is added back by InteractiveShellApp.init_path()\n" - ] - }, { "data": { "text/plain": [ - "ObjectId('5d7e6769e1eaff62f15de0d2')" + "" ] }, - "execution_count": 1, + "execution_count": 3, "metadata": {}, "output_type": "execute_result" } @@ -33,27 +25,243 @@ "\n", "client = pymongo.MongoClient (host=\"da1.eecs.utk.edu\")\n", "db = client ['fdac19mp2']\n", - "coll = db ['audris']\n", + "coll = db ['bwhite39']\n", "# for each dataset\n", - "coll.insert_one ( { 'topic':'git URLs', 'first dataset': 'largest projects', 'license': 'NA', 'description': 'The list of projects on github with the largest number of starts', 'urls': [ 'url1', 'url2' ] } )\n" + "coll.insert_one ( { 'topic':'Weather and Climate', 'title': 'US Weather History', 'license': 'CC-BY', 'description': 'History of weather in US', 'urls': [ 'https://data.world/fivethirtyeight/us-weather-history'] } )\n", + "coll.insert_one ( { 'topic':'Weather and Climate', 'title': 'Daily Weather Records', 'license': 'U.S. Government Work', 'description': 'Daily weather records across the US', 'urls': [ 'https://catalog.data.gov/dataset/daily-weather-records'] } )\n", + "coll.insert_one ( { 'topic':'Weather and Climate', 'title': 'U.S. Hourly Precipitation Data', 'license': '', 'description': 'Hourly precipitation data for United States of America, Puerto Rico, the US Virgin Islands, and various Pacific Islands', 'urls': [ 'https://data.nodc.noaa.gov/cgi-bin/iso?id=gov.noaa.ncdc:C00313'] } )\n", + "coll.insert_one ( { 'topic':'Weather and Climate', 'title': 'U.S. Monthly Climate Normals (1981-2010)', 'license': 'U.S. Government Work', 'description': 'The U.S. Monthly Climate Normals for 1981 to 2010 are 30-year averages of meteorological parameters for thousands of U.S. stations located across the 50 states, as well as U.S. territories, commonwealths,', 'urls': [ 'https://catalog.data.gov/dataset/u-s-monthly-climate-normals-1981-2010'] } )\n", + "coll.insert_one ( { 'topic':'Weather and Climate', 'title': 'Historical Hourly Weather Data 2012-2017', 'license': 'Database: Open Database, Contents: Database Contents', 'description': 'The dataset contains ~5 years of high temporal resolution (hourly measurements) data of various weather attributes for 30 US & Canadian cities, 6 Israeli cities.', 'urls': [ 'https://www.kaggle.com/selfishgene/historical-hourly-weather-data'] } )\n", + "coll.insert_one ( { 'topic':'Weather and Climate', 'title': 'U.S. 15 Minute Precipitation Data', 'license': '', 'description': ' This is precipitation data. The primary source of data for this file is approximately 2,000 mostly U.S. weather stations operated or managed by the U.S. National Weather Service', 'urls': [ 'https://data.nodc.noaa.gov/cgi-bin/iso?id=gov.noaa.ncdc:C00505'] } )\n", + "coll.insert_one ( { 'topic':'Weather and Climate', 'title': 'U.S. average annual temperature 1895-2018', 'license': '', 'description': 'U.S. average annual temperature 1895-2018', 'urls': [ 'https://www.statista.com/statistics/500472/annual-average-temperature-in-the-us/'] } )\n", + "coll.insert_one ( { 'topic':'Weather and Climate', 'title': 'Local Climatological Data (LCD) Publication', 'license': '', 'description': 'Local Climatological Data (LCD) contains summaries from major airport weather stations that include a daily account of temperature extremes, degree days, precipitation amounts and winds', 'urls': [ 'https://data.nodc.noaa.gov/cgi-bin/iso?id=gov.noaa.ncdc:C00128'] } )\n", + "coll.insert_one ( { 'topic':'Weather and Climate', 'title': 'Surface Weather Observations Monthly', 'license': 'U.S. Government Work', 'description': 'Surface Weather Observation 1001 Forms is a set of historical manuscript records for the period 1893-1948.', 'urls': [ 'https://catalog.data.gov/dataset/surface-weather-observations-daily'] } )\n", + "coll.insert_one ( { 'topic':'Weather and Climate', 'title': 'World Weather Records', 'license': '', 'description': 'Data includes monthly mean values of pressure, temperature, precipitation, and where available, station metadata notes documenting observation practices and station configurations.', 'urls': [ 'https://data.nodc.noaa.gov/cgi-bin/iso?id=gov.noaa.ncdc:C00160'] } )\n", + "coll.insert_one ( { 'topic':'Weather and Climate', 'title': 'Fire Danger and Fire Weather Records', 'license': '', 'description': 'The National Weather Service (formerly Weather Bureau) and Forest Service developed a program to track meteorological conditions conducive to forest fires, resulting in the ability to forecast and issue fire warnings and watches in national forests around the country.', 'urls': [ 'https://data.nodc.noaa.gov/cgi-bin/iso?id=gov.noaa.ncdc:C01205'] } )\n", + "coll.insert_one ( { 'topic':'Weather and Climate', 'title': 'Historical Regional Climate Data', 'license': 'Fair Usage', 'description': 'Monthly and seasonal values of: Max Temp Min Temp Mean Temp Sunshine Rainfall Raindays >=1.0mm Days of Air Frost', 'urls': [ 'https://data.gov.uk/dataset/c258aea4-83bb-40d3-95f2-1452a8db5814/historical-regional-climate-data'] } )\n", + "coll.insert_one ( { 'topic':'Weather and Climate', 'title': 'Surface Prognostic Chart', 'license': '', 'description': 'Surface Prognostic Charts are historical surface prognostic (forecast) charts created by the United States Weather Bureau. They include fronts, isobars, cloud, and precipitation areas. T', 'urls': [ 'https://data.nodc.noaa.gov/cgi-bin/iso?id=gov.noaa.ncdc:C00185'] } )\n", + "coll.insert_one ( { 'topic':'Weather and Climate', 'title': 'NOAA Next Generation Radar (NEXRAD) Level 3 Products', 'license': '', 'description': 'This dataset consists of Level 3 weather radar products collected from Next-Generation Radar (NEXRAD) stations located in the contiguous United States, Alaska, Hawaii, U.S. territories and at military base sites. NEXRAD is a network of 160 high-resolution Doppler weather radars operated by the NOAA National Weather Service (NWS), the Federal Aviation Administration (FAA), and the U.S. Air Force (USAF). Doppler radars detect atmospheric precipitation and winds, which allow scientists to track and anticipate weather events, such as rain, ice pellets, snow, hail, and tornadoes, as well as some non-weather objects like birds and insects', 'urls': [ 'https://data.nodc.noaa.gov/cgi-bin/iso?id=gov.noaa.ncdc:C00708'] } )\n", + "coll.insert_one ( { 'topic':'Weather and Climate', 'title': 'Weather Conditions in World War Two', 'license': 'U.S. Government Works', 'description': 'The dataset contains information on weather conditions recorded on each day at various weather stations around the world. Information includes precipitation, snowfall, temperatures, wind speed and whether the day included thunder storms or other poor weather conditions.', 'urls': [ 'https://www.kaggle.com/smid80/weatherww2'] } )\n", + "coll.insert_one ( { 'topic':'Weather and Climate', 'title': 'U.S. weather - number of tornadoes from 1995 to 2018', 'license': '', 'description': 'The statistic shows the number of tornadoes in the United States from 1995 to 2018. In 2018, there were a total 1,154 tornadoes reported in the United States.', 'urls': [ 'https://www.statista.com/statistics/203682/number-of-tornadoes-in-the-us-since-1995/'] } )\n", + "coll.insert_one ( { 'topic':'Weather and Climate', 'title': 'Continental United States Hurricane Strikes', 'license': 'U.S. Government Work', 'description': ' The poster includes all hurricanes that affected the U.S. since 1950 with hurricane force winds, although some did not necessarily make landfall. ', 'urls': [ 'https://catalog.data.gov/dataset/continental-united-states-hurricane-strikes'] } )\n", + "coll.insert_one ( { 'topic':'Weather and Climate', 'title': 'Worldwide historical hurricane tracks from 1848 through the previous hurricane season', 'license': '', 'description': 'This Historical Hurricane Tracks web site provides visualizations of storm tracks derived from the 6-hourly (0000, 0600, 1200, 1800 UTC) center locations and intensities for subtropical depressions and storms, extratropical storms, tropical depressions and storms, and all hurricanes, from 1848 through the previous Atlantic hurricane season (June 1 through November 30) as recorded in the International Best Track Archive for Climate Stewardship (IBTrACS, http://www.ncdc.noaa.gov/oa/ibtracs/index.php) data set.', 'urls': [ 'https://catalog.data.gov/dataset/worldwide-historical-hurricane-tracks-from-1848-through-the-previous-hurricane-season'] } )\n", + "coll.insert_one ( { 'topic':'Weather and Climate', 'title': 'Global Forecast System (GFS) [0.5 Deg.]', 'license': 'U.S. Government Work', 'description': 'The Global Forecast System (GFS) is a weather forecast model produced by the National Centers for Environmental Prediction (NCEP). Dozens of atmospheric and land-soil variables are available through this dataset, from temperatures, winds, and precipitation to soil moisture and atmospheric ozone concentration.', 'urls': [ 'https://catalog.data.gov/dataset/global-forecast-system-gfs-0-5-deg'] } )\n", + "coll.insert_one ( { 'topic':'Weather and Climate', 'title': 'Quality Controlled Local Climatological Data (QCLCD) Publication', 'license': 'MD_LegalConstraints', 'description': 'Quality Controlled Local Climatological Data (QCLCD) contains summaries from major airport weather stations that include a daily account of temperature extremes, degree days, precipitation amounts and winds.', 'urls': [ 'http://datadiscoverystudio.org/geoportal/rest/metadata/item/53b2954fb27b47eeacd94939f03c8fc3/html'] } )" ] }, { "cell_type": "code", - "execution_count": 3, + "execution_count": 5, "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ - "{'_id': ObjectId('5d7e6769e1eaff62f15de0d2'),\n", - " 'description': 'The list of projects on github with the '\n", - " 'largest number of starts',\n", - " 'first dataset': 'largest projects',\n", - " 'license': 'NA',\n", - " 'topic': 'git URLs',\n", - " 'urls': ['url1', 'url2']}\n" + "{'_id': ObjectId('5d88e18ad9d508b83c8acfb5'),\n", + " 'description': 'History of weather in US',\n", + " 'license': 'CC-BY',\n", + " 'title': 'US Weather History',\n", + " 'topic': 'Weather and Climate',\n", + " 'urls': ['https://data.world/fivethirtyeight/us-weather-history']}\n", + "{'_id': ObjectId('5d88e18ad9d508b83c8acfb6'),\n", + " 'description': 'Daily weather records across the US',\n", + " 'license': 'U.S. Government Work',\n", + " 'title': 'Daily Weather Records',\n", + " 'topic': 'Weather and Climate',\n", + " 'urls': ['https://catalog.data.gov/dataset/daily-weather-records']}\n", + "{'_id': ObjectId('5d88e18ad9d508b83c8acfb7'),\n", + " 'description': 'Hourly precipitation data for United States of '\n", + " 'America, Puerto Rico, the US Virgin Islands, '\n", + " 'and various Pacific Islands',\n", + " 'license': '',\n", + " 'title': 'U.S. Hourly Precipitation Data',\n", + " 'topic': 'Weather and Climate',\n", + " 'urls': ['https://data.nodc.noaa.gov/cgi-bin/iso?id=gov.noaa.ncdc:C00313']}\n", + "{'_id': ObjectId('5d88e18ad9d508b83c8acfb8'),\n", + " 'description': 'The U.S. Monthly Climate Normals for 1981 to '\n", + " '2010 are 30-year averages of meteorological '\n", + " 'parameters for thousands of U.S. stations '\n", + " 'located across the 50 states, as well as U.S. '\n", + " 'territories, commonwealths,',\n", + " 'license': 'U.S. Government Work',\n", + " 'title': 'U.S. Monthly Climate Normals (1981-2010)',\n", + " 'topic': 'Weather and Climate',\n", + " 'urls': ['https://catalog.data.gov/dataset/u-s-monthly-climate-normals-1981-2010']}\n", + "{'_id': ObjectId('5d88e18ad9d508b83c8acfb9'),\n", + " 'description': 'The dataset contains ~5 years of high temporal '\n", + " 'resolution (hourly measurements) data of '\n", + " 'various weather attributes for 30 US & '\n", + " 'Canadian cities, 6 Israeli cities.',\n", + " 'license': 'Database: Open Database, Contents: Database '\n", + " 'Contents',\n", + " 'title': 'Historical Hourly Weather Data 2012-2017',\n", + " 'topic': 'Weather and Climate',\n", + " 'urls': ['https://www.kaggle.com/selfishgene/historical-hourly-weather-data']}\n", + "{'_id': ObjectId('5d88e18ad9d508b83c8acfba'),\n", + " 'description': ' This is precipitation data. The primary '\n", + " 'source of data for this file is approximately '\n", + " '2,000 mostly U.S. weather stations operated or '\n", + " 'managed by the U.S. National Weather Service',\n", + " 'license': '',\n", + " 'title': 'U.S. 15 Minute Precipitation Data',\n", + " 'topic': 'Weather and Climate',\n", + " 'urls': ['https://data.nodc.noaa.gov/cgi-bin/iso?id=gov.noaa.ncdc:C00505']}\n", + "{'_id': ObjectId('5d88e18ad9d508b83c8acfbb'),\n", + " 'description': 'U.S. average annual temperature 1895-2018',\n", + " 'license': '',\n", + " 'title': 'U.S. average annual temperature 1895-2018',\n", + " 'topic': 'Weather and Climate',\n", + " 'urls': ['https://www.statista.com/statistics/500472/annual-average-temperature-in-the-us/']}\n", + "{'_id': ObjectId('5d88e18ad9d508b83c8acfbc'),\n", + " 'description': 'Local Climatological Data (LCD) contains '\n", + " 'summaries from major airport weather stations '\n", + " 'that include a daily account of temperature '\n", + " 'extremes, degree days, precipitation amounts '\n", + " 'and winds',\n", + " 'license': '',\n", + " 'title': 'Local Climatological Data (LCD) Publication',\n", + " 'topic': 'Weather and Climate',\n", + " 'urls': ['https://data.nodc.noaa.gov/cgi-bin/iso?id=gov.noaa.ncdc:C00128']}\n", + "{'_id': ObjectId('5d88e18ad9d508b83c8acfbd'),\n", + " 'description': 'Surface Weather Observation 1001 Forms is a '\n", + " 'set of historical manuscript records for the '\n", + " 'period 1893-1948.',\n", + " 'license': 'U.S. Government Work',\n", + " 'title': 'Surface Weather Observations Monthly',\n", + " 'topic': 'Weather and Climate',\n", + " 'urls': ['https://catalog.data.gov/dataset/surface-weather-observations-daily']}\n", + "{'_id': ObjectId('5d88e18ad9d508b83c8acfbe'),\n", + " 'description': 'Data includes monthly mean values of pressure, '\n", + " 'temperature, precipitation, and where '\n", + " 'available, station metadata notes documenting '\n", + " 'observation practices and station '\n", + " 'configurations.',\n", + " 'license': '',\n", + " 'title': 'World Weather Records',\n", + " 'topic': 'Weather and Climate',\n", + " 'urls': ['https://data.nodc.noaa.gov/cgi-bin/iso?id=gov.noaa.ncdc:C00160']}\n", + "{'_id': ObjectId('5d88e18ad9d508b83c8acfbf'),\n", + " 'description': 'The National Weather Service (formerly Weather '\n", + " 'Bureau) and Forest Service developed a program '\n", + " 'to track meteorological conditions conducive '\n", + " 'to forest fires, resulting in the ability to '\n", + " 'forecast and issue fire warnings and watches '\n", + " 'in national forests around the country.',\n", + " 'license': '',\n", + " 'title': 'Fire Danger and Fire Weather Records',\n", + " 'topic': 'Weather and Climate',\n", + " 'urls': ['https://data.nodc.noaa.gov/cgi-bin/iso?id=gov.noaa.ncdc:C01205']}\n", + "{'_id': ObjectId('5d88e18ad9d508b83c8acfc0'),\n", + " 'description': 'Monthly and seasonal values of: Max Temp Min '\n", + " 'Temp Mean Temp Sunshine Rainfall Raindays '\n", + " '>=1.0mm Days of Air Frost',\n", + " 'license': 'Fair Usage',\n", + " 'title': 'Historical Regional Climate Data',\n", + " 'topic': 'Weather and Climate',\n", + " 'urls': ['https://data.gov.uk/dataset/c258aea4-83bb-40d3-95f2-1452a8db5814/historical-regional-climate-data']}\n", + "{'_id': ObjectId('5d88e18ad9d508b83c8acfc1'),\n", + " 'description': 'Surface Prognostic Charts are historical '\n", + " 'surface prognostic (forecast) charts created '\n", + " 'by the United States Weather Bureau. They '\n", + " 'include fronts, isobars, cloud, and '\n", + " 'precipitation areas. T',\n", + " 'license': '',\n", + " 'title': 'Surface Prognostic Chart',\n", + " 'topic': 'Weather and Climate',\n", + " 'urls': ['https://data.nodc.noaa.gov/cgi-bin/iso?id=gov.noaa.ncdc:C00185']}\n", + "{'_id': ObjectId('5d88e18ad9d508b83c8acfc2'),\n", + " 'description': 'This dataset consists of Level 3 weather radar '\n", + " 'products collected from Next-Generation Radar '\n", + " '(NEXRAD) stations located in the contiguous '\n", + " 'United States, Alaska, Hawaii, U.S. '\n", + " 'territories and at military base sites. NEXRAD '\n", + " 'is a network of 160 high-resolution Doppler '\n", + " 'weather radars operated by the NOAA National '\n", + " 'Weather Service (NWS), the Federal Aviation '\n", + " 'Administration (FAA), and the U.S. Air Force '\n", + " '(USAF). Doppler radars detect atmospheric '\n", + " 'precipitation and winds, which allow '\n", + " 'scientists to track and anticipate weather '\n", + " 'events, such as rain, ice pellets, snow, hail, '\n", + " 'and tornadoes, as well as some non-weather '\n", + " 'objects like birds and insects',\n", + " 'license': '',\n", + " 'title': 'NOAA Next Generation Radar (NEXRAD) Level 3 Products',\n", + " 'topic': 'Weather and Climate',\n", + " 'urls': ['https://data.nodc.noaa.gov/cgi-bin/iso?id=gov.noaa.ncdc:C00708']}\n", + "{'_id': ObjectId('5d88e18ad9d508b83c8acfc3'),\n", + " 'description': 'The dataset contains information on weather '\n", + " 'conditions recorded on each day at various '\n", + " 'weather stations around the world. Information '\n", + " 'includes precipitation, snowfall, '\n", + " 'temperatures, wind speed and whether the day '\n", + " 'included thunder storms or other poor weather '\n", + " 'conditions.',\n", + " 'license': 'U.S. Government Works',\n", + " 'title': 'Weather Conditions in World War Two',\n", + " 'topic': 'Weather and Climate',\n", + " 'urls': ['https://www.kaggle.com/smid80/weatherww2']}\n", + "{'_id': ObjectId('5d88e18ad9d508b83c8acfc4'),\n", + " 'description': 'The statistic shows the number of tornadoes in '\n", + " 'the United States from 1995 to 2018. In 2018, '\n", + " 'there were a total 1,154 tornadoes reported in '\n", + " 'the United States.',\n", + " 'license': '',\n", + " 'title': 'U.S. weather - number of tornadoes from 1995 to 2018',\n", + " 'topic': 'Weather and Climate',\n", + " 'urls': ['https://www.statista.com/statistics/203682/number-of-tornadoes-in-the-us-since-1995/']}\n", + "{'_id': ObjectId('5d88e18ad9d508b83c8acfc5'),\n", + " 'description': ' The poster includes all hurricanes that '\n", + " 'affected the U.S. since 1950 with hurricane '\n", + " 'force winds, although some did not necessarily '\n", + " 'make landfall. ',\n", + " 'license': 'U.S. Government Work',\n", + " 'title': 'Continental United States Hurricane Strikes',\n", + " 'topic': 'Weather and Climate',\n", + " 'urls': ['https://catalog.data.gov/dataset/continental-united-states-hurricane-strikes']}\n", + "{'_id': ObjectId('5d88e18ad9d508b83c8acfc6'),\n", + " 'description': 'This Historical Hurricane Tracks web site '\n", + " 'provides visualizations of storm tracks '\n", + " 'derived from the 6-hourly (0000, 0600, 1200, '\n", + " '1800 UTC) center locations and intensities for '\n", + " 'subtropical depressions and storms, '\n", + " 'extratropical storms, tropical depressions and '\n", + " 'storms, and all hurricanes, from 1848 through '\n", + " 'the previous Atlantic hurricane season (June 1 '\n", + " 'through November 30) as recorded in the '\n", + " 'International Best Track Archive for Climate '\n", + " 'Stewardship (IBTrACS, '\n", + " 'http://www.ncdc.noaa.gov/oa/ibtracs/index.php) '\n", + " 'data set.',\n", + " 'license': '',\n", + " 'title': 'Worldwide historical hurricane tracks from 1848 '\n", + " 'through the previous hurricane season',\n", + " 'topic': 'Weather and Climate',\n", + " 'urls': ['https://catalog.data.gov/dataset/worldwide-historical-hurricane-tracks-from-1848-through-the-previous-hurricane-season']}\n", + "{'_id': ObjectId('5d88e18ad9d508b83c8acfc7'),\n", + " 'description': 'The Global Forecast System (GFS) is a weather '\n", + " 'forecast model produced by the National '\n", + " 'Centers for Environmental Prediction (NCEP). '\n", + " 'Dozens of atmospheric and land-soil variables '\n", + " 'are available through this dataset, from '\n", + " 'temperatures, winds, and precipitation to soil '\n", + " 'moisture and atmospheric ozone concentration.',\n", + " 'license': 'U.S. Government Work',\n", + " 'title': 'Global Forecast System (GFS) [0.5 Deg.]',\n", + " 'topic': 'Weather and Climate',\n", + " 'urls': ['https://catalog.data.gov/dataset/global-forecast-system-gfs-0-5-deg']}\n", + "{'_id': ObjectId('5d88e18ad9d508b83c8acfc8'),\n", + " 'description': 'Quality Controlled Local Climatological Data '\n", + " '(QCLCD) contains summaries from major airport '\n", + " 'weather stations that include a daily account '\n", + " 'of temperature extremes, degree days, '\n", + " 'precipitation amounts and winds.',\n", + " 'license': 'MD_LegalConstraints',\n", + " 'title': 'Quality Controlled Local Climatological Data (QCLCD) '\n", + " 'Publication',\n", + " 'topic': 'Weather and Climate',\n", + " 'urls': ['http://datadiscoverystudio.org/geoportal/rest/metadata/item/53b2954fb27b47eeacd94939f03c8fc3/html']}\n" ] } ], @@ -62,7 +270,7 @@ "import pymongo, json\n", "client = pymongo.MongoClient (host=\"da1.eecs.utk.edu\")\n", "db = client ['fdac19mp2']\n", - "coll = db ['audris']\n", + "coll = db ['bwhite39']\n", "pp = pprint.PrettyPrinter(indent=1,width=65)\n", "for r in coll. find():\n", " print(pp .pformat (r)) " @@ -94,7 +302,7 @@ "name": "python", "nbconvert_exporter": "python", "pygments_lexer": "ipython3", - "version": "3.5.2" + "version": "3.6.7" } }, "nbformat": 4,