Vivek Teega
GitHub
commit
4196b1cfd2
3
.gitignore
vendored
3
.gitignore
vendored
@ -2,4 +2,5 @@
|
||||
node_modules
|
||||
__pycache__/*
|
||||
*.pyc
|
||||
__pycache__/usernames.cpython-37.pyc
|
||||
__pycache__/usernames.cpython-37.pyc
|
||||
usercsv/
|
||||
18
README.md
18
README.md
@ -1,5 +1,19 @@
|
||||
# beautifultwint
|
||||
# Beautiful Twint
|
||||
|
||||
Beautiful Twint scrapes out data of Twitter users specified. The data is pulled using Twint module and stored as csv files. The data in the files will be visualized as per the needs of [Ranchi Mall's](https://twitter.com/ranchimallflo "What is Ranchi Mall?") marketing efforts.
|
||||
|
||||
## Pre-Requisites
|
||||
The code is written in python3 and makes use of the following python modules:
|
||||
1. twint
|
||||
2. pandas
|
||||
3. Flask
|
||||
|
||||
You can either run `pip3 install twint pandas Flask` or `pip3 install requirements.txt` to install them.
|
||||
|
||||
## Running the app
|
||||
The app performs 2 functions
|
||||
1. Fetch Twitter data of userhandles mentionded in the file `usernames.py` & store them as CSV
|
||||
2. Display
|
||||
|
||||
beautifultwint is an app which pulls out twitter information for usersmention in *usernames.py* using Twint module and visualizes them in use cases which are required for Ranchi Mall's marketing efforts.
|
||||
|
||||
Ps - This is a work in progress app as of April 23, 2020
|
||||
|
||||
24
display.py
Normal file
24
display.py
Normal file
@ -0,0 +1,24 @@
|
||||
from flask import Flask, render_template
|
||||
import pandas as pd
|
||||
import os
|
||||
|
||||
app = Flask(__name__)
|
||||
current_folder_path = os.path.abspath(os.getcwd())
|
||||
csv_foldername = 'usercsv'
|
||||
csv_folderpath = os.path.join(current_folder_path, 'usercsv')
|
||||
|
||||
@app.route('/')
|
||||
def homepage():
|
||||
# show a list of all files
|
||||
files = os.listdir(os.path.join(current_folder_path, csv_foldername))
|
||||
# read user files and display likes and tweet
|
||||
newdf = pd.DataFrame()
|
||||
for idx, filename in enumerate(files):
|
||||
userdata = pd.read_csv(os.path.join(csv_folderpath,filename), usecols=['username','likes_count','tweet'])
|
||||
newdf = newdf.append(userdata.loc[0])
|
||||
newdf = newdf[['username','likes_count','tweet']]
|
||||
return render_template('index.html', tables=[newdf.to_html(classes='data', index=False)], titles=newdf.columns.values)
|
||||
|
||||
|
||||
if __name__=='__main__':
|
||||
app.run(debug=True)
|
||||
@ -1,6 +0,0 @@
|
||||
from flask import Flask
|
||||
app = Flask(__name__)
|
||||
|
||||
@app.route('/')
|
||||
def homepage():
|
||||
return 'Welcome to beautiful twint homepage'
|
||||
25
fetchcsv.py
25
fetchcsv.py
@ -5,15 +5,34 @@
|
||||
import twint
|
||||
import os
|
||||
from usernames import *
|
||||
import shutil
|
||||
import os
|
||||
|
||||
current_folder_path = os.path.abspath(os.getcwd())
|
||||
csv_foldername = 'usercsv'
|
||||
|
||||
def fetchUserdata(username):
|
||||
c = twint.Config()
|
||||
c.Username = username
|
||||
c.Limit = 10
|
||||
c.Store_csv = True
|
||||
current_folder_path = os.path.abspath(os.getcwd())
|
||||
c.Output = os.path.join(current_folder_path, f"{username}.csv")
|
||||
twint.run.Search(c)
|
||||
|
||||
for idx, username in enumerate(twitterUsernames):
|
||||
fetchUserdata(username)
|
||||
def moveCSVfiles():
|
||||
# Twint saves all csv files in the same folder, so move all of them to usercsv/
|
||||
# todo: Find out why Twint doesn save to absolute path and remove the need for this step
|
||||
files = os.listdir(current_folder_path)
|
||||
destfolder = os.path.join(current_folder_path,csv_foldername)
|
||||
for f in files:
|
||||
if (f.endswith('.csv')):
|
||||
shutil.move(os.path.join(current_folder_path,f), os.path.join(destfolder,f))
|
||||
|
||||
if __name__=='__main__':
|
||||
# Fetch csv data and save
|
||||
for idx, username in enumerate(twitterUsernames):
|
||||
fetchUserdata(username)
|
||||
# move csv files to usercsv folder
|
||||
moveCSVfiles()
|
||||
|
||||
|
||||
|
||||
3
requirements.txt
Normal file
3
requirements.txt
Normal file
@ -0,0 +1,3 @@
|
||||
pandas
|
||||
Flask
|
||||
twint
|
||||
101
templates/index.html
Normal file
101
templates/index.html
Normal file
@ -0,0 +1,101 @@
|
||||
<html>
|
||||
<head>
|
||||
<title>Twitter user info</title>
|
||||
</head>
|
||||
|
||||
<script>
|
||||
|
||||
|
||||
body {
|
||||
font-family: 'open sans', sans-serif;
|
||||
font-size: 10pt;
|
||||
}
|
||||
|
||||
table {
|
||||
width: 100%;
|
||||
tr {
|
||||
th {
|
||||
background: #162941;
|
||||
color: white;
|
||||
text-align: left;
|
||||
}
|
||||
|
||||
td {
|
||||
overflow: hidden;
|
||||
text-overflow: ellipsis;
|
||||
max-width: 200px;
|
||||
white-space: nowrap;
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
tr:nth-child(odd) {
|
||||
background-color: rgba(171,206,227, .3);
|
||||
}
|
||||
|
||||
tr:hover {
|
||||
background-color: rgba(171,206,227, 1);
|
||||
}
|
||||
|
||||
th,td {
|
||||
padding: .5rem;
|
||||
}
|
||||
|
||||
|
||||
}
|
||||
|
||||
|
||||
@media screen and (max-width: 700px) {
|
||||
table {
|
||||
|
||||
th {
|
||||
display: none;
|
||||
}
|
||||
|
||||
tr {
|
||||
display: grid;
|
||||
grid-template-columns: auto auto;
|
||||
grid-template-areas:
|
||||
"top-left top-right"
|
||||
"bottom-left bottom-right";
|
||||
|
||||
td:first-child {
|
||||
font-size: 1em;
|
||||
}
|
||||
|
||||
td {
|
||||
padding: .1rem;
|
||||
max-width: 100%;
|
||||
overflow: hidden;
|
||||
}
|
||||
|
||||
td:nth-child(3), td:nth-child(4) {
|
||||
text-align: right;
|
||||
}
|
||||
|
||||
td.name {
|
||||
grid-area: top-left;
|
||||
}
|
||||
|
||||
td.company {
|
||||
grid-area: bottom-left;
|
||||
}
|
||||
|
||||
td.phone {
|
||||
grid-area: top-right;
|
||||
}
|
||||
|
||||
td.email {
|
||||
grid-area: bottom-right;
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
</script>
|
||||
|
||||
<body>
|
||||
<h1>Twitter Users</h1>
|
||||
{% for table in tables %} {{titles[loop.index]}} {{ table|safe }} {% endfor
|
||||
%}
|
||||
</body>
|
||||
</html>
|
||||
88
usernames.py
88
usernames.py
@ -1,27 +1,61 @@
|
||||
twitterUsernames = ["saketrx",
|
||||
"vivekteega",
|
||||
"ranchimallflo",
|
||||
"zee24taasnews",
|
||||
"alexmohajer",
|
||||
"pbns_india",
|
||||
"wtop",
|
||||
"live_hindustan",
|
||||
"kashjackson2018",
|
||||
"livemint",
|
||||
"trivworks",
|
||||
"jagranenglish",
|
||||
"tommyigoe",
|
||||
"maevemarsden",
|
||||
"anncoulter",
|
||||
"iamjohnales",
|
||||
"justin_marks_",
|
||||
"carmenbeat",
|
||||
"asranomani",
|
||||
"ralf_stegner",
|
||||
"iamjohnales",
|
||||
"iamjohnales",
|
||||
"ericg1247",
|
||||
"canoe",
|
||||
"lastampa",
|
||||
"kris6news",
|
||||
"the_hindu"]
|
||||
twitterUsernames = ["DesiCryptoHodlr",
|
||||
"VishalHKothari",
|
||||
"TabassumNaiz",
|
||||
"Shaanush",
|
||||
"bitkashyap",
|
||||
"howdy_akshay",
|
||||
"D_P_tripathi",
|
||||
"simplykashif",
|
||||
"mishralokk",
|
||||
"blockchainlaw91",
|
||||
"DaniAdvocate",
|
||||
"ThatNaimish",
|
||||
"smtgpt",
|
||||
"anshuldhir_",
|
||||
"opinderpreet",
|
||||
"manasilvora",
|
||||
"dinscrypto",
|
||||
"shineparamel",
|
||||
"amu4biz",
|
||||
"adinalini",
|
||||
"BitcoinTarun",
|
||||
"iMoneshKumar",
|
||||
"ajayjadhav",
|
||||
"Suresh81189",
|
||||
"raopreetam007",
|
||||
"chandresh1091",
|
||||
"JChittoda",
|
||||
"VenkatOnbuzz",
|
||||
"Vivek4real_",
|
||||
"aiyadt",
|
||||
"CryptoAnkush",
|
||||
"ARReddy4694",
|
||||
"Ruch_9",
|
||||
"blahblah1058",
|
||||
"Aanchal_Thakur1",
|
||||
"jafrinnnn",
|
||||
"imvijaygir",
|
||||
"cryptogiriraj",
|
||||
"RRZPX",
|
||||
"drkailaschandr1",
|
||||
"sachin_id",
|
||||
"nandubatchu",
|
||||
"therealkeerthan",
|
||||
"KunduSourodip",
|
||||
"syedvajahaath",
|
||||
"atvanguard",
|
||||
"manishgrover",
|
||||
"VaibhavMuchand3",
|
||||
"hmalviya9",
|
||||
"CryptoShrikar",
|
||||
"kg_Cashaa",
|
||||
"sidsverma",
|
||||
"darshanbathija",
|
||||
"AnkittGaur",
|
||||
"TarushaM",
|
||||
"Dul_dul",
|
||||
"AraBalaghi",
|
||||
"TheMohitMadan",
|
||||
"vaibhavchellani",
|
||||
"chetanb_",
|
||||
"HeyVixon"]
|
||||
|
||||
Loading…
Reference in New Issue
Block a user