Coursera: Automating Real-World Tasks with Python

Recently completed the Capstone of the Google IT Automation with Python Coursera Specialisation. Being a beginner user of Python, the week 4 assessment took me almost the full allocated 120mins.

Hope to provide some guidance by sharing the final scripts I used, for reference purposes. Remember to replace the user name with your qwiklabs user name. 

The real learning comes from trying and troubleshooting your scripts! (tracebacks, tracebacks, tracebacks.....)

Assessment Overview 

You work for an online fruits store, and you need to develop a system that will update the catalog information with data provided by your suppliers. The suppliers send the data as large images with an associated description of the products in two files (.TIF for the image and .txt for the description). The images need to be converted to smaller jpeg images and the text needs to be turned into an HTML file that shows the image and the product description. The contents of the HTML file need to be uploaded to a web service that is already running using Django. You also need to gather the name and weight of all fruits from the .txt files and use a Python request to upload it to your Django server.

You will create a Python script that will process the images and descriptions and then update your company's online website to add the new products.

Once the task is complete, the supplier should be notified with an email that indicates the total weight of fruit (in lbs) that were uploaded. The email should have a PDF attached with the name of the fruit and its total weight (in lbs).

Finally, in parallel to the automation running, we want to check the health of the system and send an email if something goes wrong.

What you’ll do

• Write a script that summarizes and processes sales data into different categories
• Generate a PDF using Python
• Automatically send a PDF by email
• Write a script to check the health status of the system

My Submitted Script

Script 1 changeImage.py

#!/usr/bin/env python3

import os, sys

from PIL import Image

user = os.getenv('USER')

image_directory = '/home/{}/supplier-data/images/'.format(user)

for image_name in os.listdir(image_directory):

  if not image_name.startswith('.') and 'tiff' in image_name:

    image_path = image_directory + image_name

    path = os.path.splitext(image_path)[0]

    im = Image.open(image_path)

    new_path = '{}.jpeg'.format(path)

    im.convert('RGB').resize((600,400)).save(new_path, "JPEG")

Script 2 supplier_image_upload.py

#!/usr/bin/env python3

import requests, os

url = "http://localhost/upload/"

USER = os.getenv('USER')

image_directory = '/home/{}/supplier-data/images/'.format(USER)

files = os.listdir(image_directory)

for image_name in files:

  if not image_name.startswith('.') and 'jpeg' in image_name:

    image_path = image_directory + image_name

    with open(image_path, 'rb') as opened:

      r = requests.post(url, files={'file': opened})

Script 3 run.py

#! /usr/bin/env python3

import os

import requests

import json

def catalog_data(url,description_dir):

  fruit={}

  for item in os.listdir(description_dir):

    fruit.clear()

    filename=os.path.join(description_dir,item)

    with open(filename) as f:

      line=f.readlines()

      description=""

      for i in range(2,len(line)):

        description=description+line[i].strip('\n').replace(u'\xa0',u'')

        fruit["description"]=description

        fruit["weight"]=int(line[1].strip('\n').strip('lbs'))

        fruit["name"]=line[0].strip('\n')

        fruit["image_name"]=(item.strip('.txt'))+'.jpeg'

        print(fruit)

        if url!="":

          response=requests.post(url, json=fruit)

          print(response.request.url)

          print(response.status_code)

  return 0

if __name__=='__main__':

  url='http://localhost/fruits/'

  user=os.getenv('USER')

  description_directory='/home/{}/supplier-data/descriptions/'.format(user)

  catalog_data(url,description_directory)

Script 4 reports.py

#!/usr/bin/env python3

from reportlab.platypus import Paragraph, Spacer, Image, SimpleDocTemplate

from reportlab.lib.styles import getSampleStyleSheet

def generate_report(file, title, add_info):

  styles = getSampleStyleSheet()

  report = SimpleDocTemplate(file)

  report_title = Paragraph(title, styles['h1'])

  report_info = Paragraph(add_info, styles['BodyText'])

  empty_line =  Spacer(1,20)

  report.build([report_title, empty_line, report_info, empty_line])

Script 5 report_email.py

#!/usr/bin/env python3

import datetime

import os

from run import catalog_data

from reports import generate_report

from emails import generate_email, send_email

def pdf_body(input_for,desc_dir):

  res = []

  wt = []

  for item in os.listdir(desc_dir):

    filename=os.path.join(desc_dir,item)

    with open(filename) as f:

      line=f.readlines()

      weight=line[1].strip('\n')

      name=line[0].strip('\n')

      print(name,weight)

      res.append('name: ' +name)

      wt.append('weight: ' +weight)

      print(res)

      print(wt)

  new_obj = ""

  for i in range(len(res)):

    if res[i] and input_for == 'pdf':

      new_obj += res[i] + '<br />' + wt[i] + '<br />' + '<br />'

  return new_obj

if __name__ == "__main__":

  user = os.getenv('USER')

  description_directory = '/home/{}/supplier-data/descriptions/'.format(user)

  current_date = datetime.date.today().strftime("%B %d, %Y")

  title = 'Processed Update on ' + str(current_date)

  generate_report('/tmp/processed.pdf', title, pdf_body('pdf',description_directory))

  email_subject = 'Upload Completed - Online Fruit Store'

  email_body = 'All fruits are uploaded to our website successfully. A detailed list is attached to this email'

  msg = generate_email("automation@example.com", "yourstudentusername@example.com".format(user), email_subject, email_body, "/tmp/processed.pdf")

  send_email(msg)

Script 6 emails.py

#!/usr/bin/env python 3

import email

import mimetypes

import smtplib

import os

def generate_email(sender, recipient, subject, body, attachment_path):

  message = email.message.EmailMessage()

  message["From"] = sender

  message["To"] = recipient

  message["Subject"] = subject

  message.set_content(body)

  if not attachment_path == "":

    attachment_filename = os.path.basename(attachment_path)

    mime_type, _ = mimetypes.guess_type(attachment_path)

    mime_type, mime_subtype = mime_type.split('/', 1)

    with open(attachment_path, 'rb') as ap:

      message.add_attachment(ap.read(), maintype=mime_type, subtype=mime_subtype, filename=attachment_filename)

  return message

def send_email(message):

  mail_server = smtplib.SMTP('localhost')

  mail_server.send_message(message)

  mail_server.quit()

Script 7 health_check.py

#!/usr/bin/env python3

import socket

import shutil

import psutil

import emails

def check_localhost():

  localhost = socket.gethostbyname('localhost')

  return localhost== "127.0.0.1"

def check_disk_usage(disk):

  du = shutil.disk_usage(disk)

  free = du.free / du.total * 100

  return free > 20

def check_memory_usage():

  mu = psutil.virtual_memory().available

  total = mu / (1024.0 ** 2)

  return total > 500

def check_cpu_usage():

  usage = psutil.cpu_percent(1)

  return usage < 80

def send_email(subject):

  email = emails.generate_email("automation@example.com", "yourstudentusername@example.com", subject, "Please check your system and resolve the issue as soon as practicable", "")

  emails.send_email(email)

if not check_cpu_usage() :

  subject="Error - CPU usage is over 80%"

  print(subject)

  send_email(subject)

if not check_memory_usage():

  subject = "Error - Available memory is less than 500MB"

  print(subject)

if not check_disk_usage('/') :

  subject = "Error - Available disk space is less than 20%"

  print(subject)

  send_email(subject)

if not check_localhost():

  subject = "Error - localhost cannot be resolved to 127.0.0.1"

  print(subject)

  send_email(subject)

Hope this was useful for your self-improvement quest!