You can not select more than 25 topics
Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.
352 lines
19 KiB
352 lines
19 KiB
#!/usr/bin/env python
|
|
# encoding: utf-8
|
|
|
|
import sys
|
|
import requests
|
|
from Naked.settings import debug as DEBUG_FLAG
|
|
|
|
#------------------------------------------------------------------------------
|
|
#[ HTTP class]
|
|
# handle HTTP requests
|
|
# Uses the requests external library to handle HTTP requests and response object (available on PyPI)
|
|
#------------------------------------------------------------------------------
|
|
class HTTP():
|
|
def __init__(self, url="", request_timeout=10):
|
|
self.url = url
|
|
self.request_timeout = request_timeout
|
|
#------------------------------------------------------------------------------
|
|
# HTTP response properties (assignment occurs with the HTTP request methods)
|
|
#------------------------------------------------------------------------------
|
|
self.res = None # assigned with the requests external library response object after a HTTP method call
|
|
|
|
#------------------------------------------------------------------------------
|
|
# [ get method ] (string) -
|
|
# HTTP GET request - returns text string
|
|
# returns data stream read from the URL (string)
|
|
# Default timeout = 10 s from class constructor
|
|
# Re-throws ConnectionError on failed connection (e.g. no site at URL)
|
|
# Test : test_NETWORK.py :: test_http_get, test_http_get_response_change,
|
|
# test_http_post_reponse_change, test_http_get_response_check
|
|
#------------------------------------------------------------------------------
|
|
def get(self, follow_redirects=True):
|
|
try:
|
|
response = requests.get(self.url, timeout=self.request_timeout, allow_redirects=follow_redirects)
|
|
self.res = response # assign the response object from requests to a property on the instance of HTTP class
|
|
return response.text
|
|
except requests.exceptions.ConnectionError as ce:
|
|
return False
|
|
except Exception as e:
|
|
if DEBUG_FLAG:
|
|
sys.stderr.write("Naked Framework Error: Unable to perform GET request with the URL " + self.url + " using the get() method (Naked.toolshed.network.py)")
|
|
raise e
|
|
|
|
#------------------------------------------------------------------------------
|
|
# [ get_data method ] (binary data)
|
|
# HTTP GET request, return binary data
|
|
# returns data stream with raw binary data
|
|
#------------------------------------------------------------------------------
|
|
def get_bin(self):
|
|
try:
|
|
response = requests.get(self.url, timeout=self.request_timeout)
|
|
self.res = response # assign the response object from requests to a property on the instance
|
|
return response.content # return binary data instead of text (get() returns text)
|
|
except requests.exceptions.ConnectionError as ce:
|
|
return False
|
|
except Exception as e:
|
|
if DEBUG_FLAG:
|
|
sys.stderr.write("Naked Framework Error: Unable to perform GET request with the URL " + self.url + " using the get_data() method (Naked.toolshed.network.py)")
|
|
raise e
|
|
|
|
#------------------------------------------------------------------------------
|
|
# [ get_bin_write_file method ] (boolean)
|
|
# open HTTP data stream with GET request, make file with the returned binary data
|
|
# file path is passed to the method by the developer
|
|
# set suppress_output to True if you want to suppress the d/l status information that is printed to the standard output stream
|
|
# return True on successful pull and write to disk
|
|
# Tests: test_NETWORK.py :: test_http_get_binary
|
|
#------------------------------------------------------------------------------
|
|
def get_bin_write_file(self, filepath="", suppress_output = False, overwrite_existing = False):
|
|
try:
|
|
import os # used for os.fsync() method in the write
|
|
# Confirm that the file does not exist and prevent overwrite if it does (unless developer indicates otherwise)
|
|
if not overwrite_existing:
|
|
from Naked.toolshed.system import file_exists
|
|
if file_exists(filepath):
|
|
if not suppress_output:
|
|
print("Download aborted. A local file with the requested filename exists on the path.")
|
|
return False
|
|
if (filepath == "" and len(self.url) > 1):
|
|
filepath = self.url.split('/')[-1] # use the filename from URL and working directory as default if not specified
|
|
if not suppress_output:
|
|
sys.stdout.write("Downloading file from " + self.url + "...")
|
|
sys.stdout.flush()
|
|
response = requests.get(self.url, timeout=self.request_timeout, stream=True)
|
|
self.res = response
|
|
with open(filepath, 'wb') as f: # write as binary data
|
|
for chunk in response.iter_content(chunk_size=2048):
|
|
f.write(chunk)
|
|
f.flush()
|
|
os.fsync(f.fileno()) # flush all internal buffers to disk
|
|
if not suppress_output:
|
|
print(" ")
|
|
print("Download complete.")
|
|
return True # return True if successful write
|
|
except requests.exceptions.ConnectionError as ce:
|
|
return False
|
|
except Exception as e:
|
|
if DEBUG_FLAG:
|
|
sys.stderr.write("Naked Framework Error: Unable to perform GET request and write file with the URL " + self.url + " using the get_bin_write_file() method (Naked.toolshed.network.py)")
|
|
raise e
|
|
|
|
#------------------------------------------------------------------------------
|
|
# [ get_txt_write_file method ] (boolean)
|
|
# open HTTP data stream with GET request, write file with utf-8 encoded text using returned text data
|
|
# file path is passed to the method by the developer (default is the base filename in the URL if not specified)
|
|
# return True on successful pull and write to disk
|
|
# Tests: test_NETWORK.py :: test_http_get_text
|
|
#------------------------------------------------------------------------------
|
|
def get_txt_write_file(self, filepath="", suppress_output = False, overwrite_existing = False):
|
|
try:
|
|
import os # used for os.fsync() method in the write
|
|
# Confirm that the file does not exist and prevent overwrite if it does (unless developer indicates otherwise)
|
|
if not overwrite_existing:
|
|
from Naked.toolshed.system import file_exists
|
|
if file_exists(filepath):
|
|
if not suppress_output:
|
|
print("Download aborted. A local file with the requested filename exists on the path.")
|
|
return False
|
|
if (filepath == "" and len(self.url) > 1):
|
|
filepath = self.url.split('/')[-1] # use the filename from URL and working directory as default if not specified
|
|
if not suppress_output:
|
|
sys.stdout.write("Downloading file from " + self.url + "...")
|
|
sys.stdout.flush()
|
|
response = requests.get(self.url, timeout=self.request_timeout, stream=True)
|
|
self.res = response
|
|
import codecs
|
|
with codecs.open(filepath, mode='w', encoding="utf-8") as f: #write as text
|
|
for chunk in response.iter_content(chunk_size=2048):
|
|
chunk = chunk.decode('utf-8')
|
|
f.write(chunk)
|
|
f.flush()
|
|
os.fsync(f.fileno()) # flush all internal buffers to disk
|
|
if not suppress_output:
|
|
print(" ")
|
|
print("Download complete.")
|
|
return True # return True if successful write
|
|
except requests.exceptions.ConnectionError as ce:
|
|
return False
|
|
except Exception as e:
|
|
if DEBUG_FLAG:
|
|
sys.stderr.write("Naked Framework Error: Unable to perform GET request and write file with the URL " + self.url + " using the get_data_write_txt() method (Naked.toolshed.network.py)")
|
|
raise e
|
|
|
|
#------------------------------------------------------------------------------
|
|
# [ head method ] (dictionary of strings)
|
|
# HTTP HEAD request
|
|
# returns a dictionary of the header strings
|
|
# test for a specific header on either the response dictionary or the instance res property
|
|
# Usage example:
|
|
# content_type = instance.res['content-type']
|
|
# Tests: test_NETWORK.py :: test_http_head
|
|
#------------------------------------------------------------------------------
|
|
def head(self):
|
|
try:
|
|
response = requests.head(self.url, timeout=self.request_timeout)
|
|
self.res = response # assign the response object from requests to a property on the instance of HTTP class
|
|
return response.headers
|
|
except requests.exceptions.ConnectionError as ce:
|
|
return False
|
|
except Exception as e:
|
|
if DEBUG_FLAG:
|
|
sys.stderr.write("Naked Framework Error: Unable to perform a HEAD request with the head() method (Naked.toolshed.network.py).")
|
|
raise e
|
|
|
|
|
|
#------------------------------------------------------------------------------
|
|
# [ post method ] (string)
|
|
# HTTP POST request for text
|
|
# returns text from the URL as a string
|
|
#------------------------------------------------------------------------------
|
|
def post(self, follow_redirects=True):
|
|
try:
|
|
response = requests.post(self.url, timeout=self.request_timeout, allow_redirects=follow_redirects)
|
|
self.res = response # assign the response object from requests to a property on the instance of HTTP class
|
|
return response.text
|
|
except requests.exceptions.ConnectionError as ce:
|
|
return False
|
|
except Exception as e:
|
|
if DEBUG_FLAG:
|
|
sys.stderr.exit("Naked Framework Error: Unable to perform a POST request with the post() method (Naked.toolshed.network.py).")
|
|
raise e
|
|
|
|
#------------------------------------------------------------------------------
|
|
# [ post_bin method ] (binary data)
|
|
# HTTP POST request for binary data
|
|
# returns binary data from the URL
|
|
#------------------------------------------------------------------------------
|
|
def post_bin(self):
|
|
try:
|
|
response = requests.post(self.url, timeout=self.request_timeout)
|
|
self.res = response # assign the response object from requests to a property on the instance of HTTP class
|
|
return response.content
|
|
except requests.exceptions.ConnectionError as ce:
|
|
return False
|
|
except Exception as e:
|
|
if DEBUG_FLAG:
|
|
sys.stderr.exit("Naked Framework Error: Unable to perform POST request with the post_bin() method (Naked.toolshed.network.py).")
|
|
raise e
|
|
|
|
#------------------------------------------------------------------------------
|
|
# [ post_bin_write_file method ] (boolean = success of write)
|
|
# HTTP POST request, write binary file with the response data
|
|
# default filepath is the basename of the URL file, may be set by passing an argument to the method
|
|
# returns a boolean that indicates the success of the file write
|
|
#------------------------------------------------------------------------------
|
|
def post_bin_write_file(self, filepath="", suppress_output = False, overwrite_existing = False):
|
|
try:
|
|
import os # used for os.fsync() method in the write
|
|
# Confirm that the file does not exist and prevent overwrite if it does (unless developer indicates otherwise)
|
|
if not overwrite_existing:
|
|
from Naked.toolshed.system import file_exists
|
|
if file_exists(filepath):
|
|
if not suppress_output:
|
|
print("Download aborted. A local file with the requested filename exists on the path.")
|
|
return False
|
|
if (filepath == "" and len(self.url) > 1):
|
|
filepath = self.url.split('/')[-1] # use the filename from URL and working directory as default if not specified
|
|
if not suppress_output:
|
|
sys.stdout.write("Downloading file from " + self.url + "...") #provide information about the download to user
|
|
sys.stdout.flush()
|
|
response = requests.post(self.url, timeout=self.request_timeout, stream=True)
|
|
self.res = response
|
|
with open(filepath, 'wb') as f: # write as binary data
|
|
for chunk in response.iter_content(chunk_size=2048):
|
|
f.write(chunk)
|
|
f.flush()
|
|
os.fsync(f.fileno()) # flush all internal buffers to disk
|
|
if not suppress_output:
|
|
print(" ")
|
|
print("Download complete.") # provide user with completion information
|
|
return True # return True if successful write
|
|
except requests.exceptions.ConnectionError as ce:
|
|
return False
|
|
except Exception as e:
|
|
if DEBUG_FLAG:
|
|
sys.stderr.write("Naked Framework Error: Unable to perform POST request and write file with the URL " + self.url + " using the post_data_write_bin() method (Naked.toolshed.network.py)")
|
|
raise e
|
|
|
|
#------------------------------------------------------------------------------
|
|
# [ post_txt_write_file method ] (boolean = success of file write)
|
|
# HTTP POST request, write utf-8 encoded text file with the response data
|
|
# default filepath is the basename of the URL file, may be set by passing an argument to the method
|
|
# returns a boolean that indicates the success of the file write
|
|
#------------------------------------------------------------------------------
|
|
def post_txt_write_file(self, filepath="", suppress_output = False, overwrite_existing = False):
|
|
try:
|
|
import os # used for os.fsync() method in the write
|
|
# Confirm that the file does not exist and prevent overwrite if it does (unless developer indicates otherwise)
|
|
if not overwrite_existing:
|
|
from Naked.toolshed.system import file_exists
|
|
if file_exists(filepath):
|
|
if not suppress_output:
|
|
print("Download aborted. A local file with the requested filename exists on the path.")
|
|
return False
|
|
if (filepath == "" and len(self.url) > 1):
|
|
filepath = self.url.split('/')[-1] # use the filename from URL and working directory as default if not specified
|
|
if not suppress_output:
|
|
sys.stdout.write("Downloading file from " + self.url + "...") #provide information about the download to user
|
|
sys.stdout.flush()
|
|
response = requests.post(self.url, timeout=self.request_timeout, stream=True)
|
|
self.res = response
|
|
import codecs
|
|
with codecs.open(filepath, mode='w', encoding="utf-8") as f: # write as binary data
|
|
for chunk in response.iter_content(chunk_size=2048):
|
|
chunk = chunk.decode('utf-8')
|
|
f.write(chunk)
|
|
f.flush()
|
|
os.fsync(f.fileno()) # flush all internal buffers to disk
|
|
if not suppress_output:
|
|
print(" ")
|
|
print("Download complete.") # provide user with completion information
|
|
return True # return True if successful write
|
|
except requests.exceptions.ConnectionError as ce:
|
|
return False
|
|
except Exception as e:
|
|
if DEBUG_FLAG:
|
|
sys.stderr.write("Naked Framework Error: Unable to perform POST request and write file with the URL " + self.url + " using the post_data_write_bin() method (Naked.toolshed.network.py)")
|
|
raise e
|
|
|
|
#------------------------------------------------------------------------------
|
|
# [ response method ]
|
|
# getter method for the requests library object that is assigned as a property
|
|
# on the HTTP class after a HTTP request method is run (e.g. get())
|
|
# Note: must run one of the HTTP request verbs to assign this property before use of getter (=None by default)
|
|
# Tests: test_NETWORK.py :: test_http_get_response_check
|
|
#------------------------------------------------------------------------------
|
|
def response(self):
|
|
try:
|
|
return self.res
|
|
except Exception as e:
|
|
if DEBUG_FLAG:
|
|
sys.stderr.write("Naked Framework Error: Unable to return the response from your HTTP request with the response() method (Naked.toolshed.network.py).")
|
|
raise e
|
|
|
|
#------------------------------------------------------------------------------
|
|
# [ get_status_ok method ] (boolean)
|
|
# return boolean whether HTTP response was in 200 status code range for GET request
|
|
# Note: this method runs its own GET request, does not need to be run separately
|
|
# Tests: test_NETWORK.py ::
|
|
#------------------------------------------------------------------------------
|
|
def get_status_ok(self):
|
|
try:
|
|
self.get() #run the get request
|
|
if self.res and self.res.status_code:
|
|
return (self.res.status_code == requests.codes.ok)
|
|
else:
|
|
return False
|
|
except requests.exceptions.ConnectionError as ce:
|
|
return False
|
|
except Exception as e:
|
|
if DEBUG_FLAG:
|
|
sys.stderr.write("Naked Framework Error: Unable to obtain the HTTP status with the get_status_ok() method (Naked.toolshed.network.py).")
|
|
raise e
|
|
|
|
#------------------------------------------------------------------------------
|
|
# [ post_status_ok method ] (boolean)
|
|
# return boolean whether HTTP response was in 200 status code range for POST request
|
|
# Note: method runs its own post method, not necessary to run separately
|
|
#------------------------------------------------------------------------------
|
|
def post_status_ok(self):
|
|
try:
|
|
self.post() #run the post request
|
|
if self.res and self.res.status_code:
|
|
return (self.res.status_code == requests.codes.ok)
|
|
else:
|
|
return False
|
|
except requests.exceptions.ConnectionError as ce:
|
|
return False
|
|
except Exception as e:
|
|
if DEBUG_FLAG:
|
|
sys.stderr.write("Naked Framework Error: Unable to obtain the HTTP status with the post_status_ok() method (Naked.toolshed.network.py).")
|
|
raise e
|
|
|
|
if __name__ == '__main__':
|
|
pass
|
|
#------------------------------------------------------------------------------
|
|
# HTTP GET 1
|
|
#------------------------------------------------------------------------------
|
|
# http = HTTP("http://www.google.com")
|
|
# data = http.get()
|
|
# print(data)
|
|
# from Naked.toolshed.file import FileWriter
|
|
# w = FileWriter("testfile.txt")
|
|
# w.write_utf8(data)
|
|
#------------------------------------------------------------------------------
|
|
# HTTP GET 2
|
|
#------------------------------------------------------------------------------
|
|
# http = HTTP()
|
|
# http.url = "http://www.google.com"
|
|
# print(http.get())
|
|
|
|
|