Having trouble extracting dynamic div list while scrolling down using Webdriver (selenium & python)

See also questions close to this topic

• Stacking model using iris dataset

  In the below code i implemented a stacking model using KNN, SVM and RF as base models with an example on iris dataset the difficulty i faced is that once i use a metamodel to stack the results such as SVM or RF or another models they gives the same results which is weird and abnormal. May anyone advise.

  from sklearn import datasets
  cancer=datasets.load_breast_cancer()
  X=cancer.data
  y=cancer.target
  
  from sklearn import datasets
  from sklearn.model_selection import train_test_split
  from sklearn.model_selection import GridSearchCV
  from sklearn.metrics import classification_report
  from sklearn.svm import SVC
  from sklearn.metrics import accuracy_score
  from sklearn.model_selection import *
  from time import *
  from sklearn import metrics
  np.random.seed(123)
  # Split the dataset in two equal parts
  X_train, X_test, y_train, y_test = train_test_split(
      X, y, test_size=0.3, random_state=None)
  
  # Set the parameters by cross-validation
  tuned_parameters =  [{'kernel': ['rbf'], 'gamma': [1e-2, 1e-3, 1e-4, 1e-5],
                       'C': [0.001, 0.10, 0.1, 10, 25, 50, 100, 1000]},
                      {'kernel': ['sigmoid'], 'gamma': [1e-2, 1e-3, 1e-4, 1e-5],
                       'C': [0.001, 0.10, 0.1, 10, 25, 50, 100, 1000] },{'kernel': ['linear'], 'C': [0.001, 0.10, 0.1, 10, 25, 50, 100, 1000]}]              
  
  
  
  
  
  print()
  
  clf = GridSearchCV(SVC(), tuned_parameters, cv=10,
                         scoring='accuracy')
  t0 = time()
  
  clf.fit(X_train, y_train)
  t = time() - t0
  print("Best parameters set found on development set:")
  print()
  print(clf.best_params_)
  print()
  print('Training accuracy')
  print(clf.best_score_)
  print(clf.best_estimator_)
  print()
  print()
  print('****Results****')
  svm_pred=cross_val_predict(clf.best_estimator_,X_train,y_train,cv=10)
  #print("\t\taccuracytrainkfold: {}".format(metrics.accuracy_score(y_train, svm_pred)))
  print("=" * 52)
  print("time cost: {}".format(t))
  #print(clf.cv_results_)
  print()
  #print(cross_val_score(clf,X_train,y_train,cv=10).mean())
  print("\t\taccuracy: {}".format(metrics.accuracy_score(y_train, svm_pred)))
  Best parameters set found on development set:
  
  {'C': 50, 'gamma': 0.01, 'kernel': 'rbf'}
  
  Training accuracy
  0.8165938864628821
  SVC(C=50, cache_size=200, class_weight=None, coef0=0.0,
    decision_function_shape='ovr', degree=3, gamma=0.01, kernel='rbf',
    max_iter=-1, probability=False, random_state=None, shrinking=True,
    tol=0.001, verbose=False)
  
  
  ****Results****
  ====================================================
  time cost: 8.607818841934204
  
          accuracy: 0.8165938864628821
  In [28]:
  opt_svm_pred=clf.best_estimator_.predict(X_test)
  print()
  print("confusion matrix\n", metrics.confusion_matrix(y_test, opt_svm_pred))
  print()
  print("\t\taccuracy: {}".format(metrics.accuracy_score(y_test, opt_svm_pred)))
  print("\t\troc_auc_score: {}".format(metrics.roc_auc_score(y_test, opt_svm_pred)))
  print("\t\tcohen_kappa_score: {}".format(metrics.cohen_kappa_score(y_test, opt_svm_pred)))
  print()
  print("\t\tclassification report")
  print("-" * 52)
  print(metrics.classification_report(y_test, opt_svm_pred)) 
  confusion matrix
   [[45  8]
   [ 6 40]]
  
          accuracy: 0.8585858585858586
          roc_auc_score: 0.8593109105824446
          cohen_kappa_score: 0.7165644171779141
  
          classification report
  ----------------------------------------------------
               precision    recall  f1-score   support
  
            0       0.88      0.85      0.87        53
            1       0.83      0.87      0.85        46
  
  avg / total       0.86      0.86      0.86        99
  
  RF¶
  In [19]:
  from sklearn.model_selection import *
  from sklearn.ensemble import RandomForestClassifier
  #making the instance
  np.random.seed(123)
  # Split the dataset in two equal parts
  X_train, X_test, y_train, y_test = train_test_split( X, y, test_size=0.3, random_state=None)
  model=RandomForestClassifier()
  #hyper parameters set
  params = {'criterion':['gini','entropy'],
            'n_estimators':[10,15,20,25,30],
            'min_samples_leaf':[1,2,3],
            'min_samples_split':[3,4,5,6,7],
            'random_state':[123],
            'n_jobs':[-1]}
  #Making models with hyper parameters sets
  RF_grid = GridSearchCV(model, param_grid=params,cv=10, n_jobs=-1)
  #learning
  t0 = time()
  RF_grid.fit(X_train,y_train)
  t0 = time()-t0
  #The best hyper parameters set
  print("Best Hyper Parameters:\n",RF_grid.best_params_)
  print("training accuracy:\n",RF_grid.best_score_)
  #Prediction
  RF_prediction=cross_val_predict(RF_grid.best_estimator_,X_train,y_train,cv=10)
  #print("\t\taccuracytrainkfold: {}".format(metrics.accuracy_score(y_train, svm_pred)))
  print("=" * 52)
  print("time cost: {}".format(t))
  #print(clf.cv_results_)
  print()
  #print(cross_val_score(clf,X_train,y_train,cv=10).mean())
  print("\t\taccuracy: {}".format(metrics.accuracy_score(y_train, RF_prediction)))
  Best Hyper Parameters:
   {'criterion': 'gini', 'min_samples_leaf': 1, 'min_samples_split': 5, 'n_estimators': 30, 'n_jobs': -1, 'random_state': 123}
  training accuracy:
   0.8558951965065502
  ====================================================
  time cost: 326.25868034362793
  
          accuracy: 0.8558951965065502
  In [25]:
  opt_RF_pred=RF_grid.best_estimator_.predict(X_test)
  #evaluation(Confusion Matrix)
  #importing the metrics module
  from sklearn import metrics
  #evaluation(Accuracy)
  print("Accuracy:",metrics.accuracy_score(opt_RF_pred,y_test))
  #evaluation(Confusion Matrix)
  print("Confusion Matrix:\n",metrics.confusion_matrix(opt_RF_pred,y_test))
  print("\t\tclassification report")
  print("-" * 52)
  print(metrics.classification_report(opt_RF_pred,y_test)) 
  Accuracy: 0.8383838383838383
  Confusion Matrix:
   [[40  3]
   [13 43]]
          classification report
  ----------------------------------------------------
               precision    recall  f1-score   support
  
            0       0.75      0.93      0.83        43
            1       0.93      0.77      0.84        56
  
  avg / total       0.86      0.84      0.84        99
  
  KNN¶
  In [21]:
  from sklearn.model_selection import GridSearchCV
  from sklearn.neighbors import KNeighborsClassifier
  import matplotlib.pyplot as plt
  from sklearn.model_selection import *
  from sklearn import metrics
  from time import *
  np.random.seed(123)
  # split into train and test
  X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.3, random_state=None)
  # instantiate model
  knn = KNeighborsClassifier()
  # create a parameter grid: map the parameter names to the values that should be searched
  # simply a python dictionary
  # key: parameter name
  # value: list of values that should be searched for that parameter
  # single key-value pair for param_grid
  param_grid = {'n_neighbors':[1,2,3,4,5,6,7,8,9,10],'leaf_size':[1,2,3,5],
            'weights':['uniform', 'distance'],
            'algorithm':['auto'],
            'n_jobs':[-1]}
  
  # instantiate the grid
  grid_knn = GridSearchCV(knn, param_grid, cv=10, scoring='accuracy')
  # fit the grid with data
  t0=time()
  grid_knn.fit(X_train, y_train)
  t = time() - t0
  # view the complete results (list of named tuples)
  #The best hyper parameters set
  print("Best Hyper Parameters:\n",grid_knn.best_params_)
  print("training accuracy:\n",grid_knn.best_score_)
  #Prediction
  knn_prediction=cross_val_predict(grid_knn.best_estimator_,X_train,y_train,cv=10)
  
  print("=" * 52)
  print("time cost: {}".format(t))
  print()
  #{'algorithm': 'auto', 'leaf_size': 1, 'n_jobs': -1, 'n_neighbors': 8, 'weights': 'uniform'}
  #print("\t\taccuracytrainkfold: {}".format(metrics.accuracy_score(y_train, svm_pred)))
  print("=" * 52)
  print("time cost: {}".format(t))
  #print(clf.cv_results_)
  print()
  #print(cross_val_score(clf,X_train,y_train,cv=10).mean())
  print("\t\taccuracy: {}".format(metrics.accuracy_score(y_train, knn_prediction)))
  Best Hyper Parameters:
   {'algorithm': 'auto', 'leaf_size': 1, 'n_jobs': -1, 'n_neighbors': 3, 'weights': 'uniform'}
  training accuracy:
   0.7991266375545851
  ====================================================
  time cost: 186.73932242393494
  
  ====================================================
  time cost: 186.73932242393494
  
          accuracy: 0.7991266375545851
  In [24]:
  opt_knn_pred=grid_knn.best_estimator_.predict(X_test)
  #importing the metrics module
  from sklearn import metrics
  #evaluation(Accuracy)
  print("Accuracy:",metrics.accuracy_score(opt_knn_pred,y_test))
  #evaluation(Confusion Metrix)
  print("Confusion Matrix:\n",metrics.confusion_matrix(opt_knn_pred,y_test))
  print()
  print("\t\taccuracy: {}".format(metrics.accuracy_score(opt_knn_pred,y_test)))
  print(metrics.classification_report(opt_knn_pred,y_test))
  Accuracy: 0.8181818181818182
  Confusion Matrix:
   [[41  6]
   [12 40]]
  
          accuracy: 0.8181818181818182
               precision    recall  f1-score   support
  
            0       0.77      0.87      0.82        47
            1       0.87      0.77      0.82        52
  
  avg / total       0.82      0.82      0.82        99
  
  PNN STACKING¶
  In [29]:
  X_pred=np.array([svm_pred,knn_prediction,RF_prediction]).T
  #pred=np.array([pred1,pred2,pred3,pred4]).T
  #output layer containg y_test 
  y_train
  #split dataset
  from sklearn.neural_network import MLPClassifier
  from sklearn.model_selection import train_test_split
  from sklearn.model_selection import GridSearchCV
  from time import *
  from neupy.algorithms import PNN
  from time import *
  np.random.seed(123)
  # Set the parameters by cross-validation
  tuned_parameters ={'std':[0.05,0.1,0.15,0.2,0.25,0.3,0.35,0.4,0.45,0.5,0.55,0.6,0.65,0.7,0.75,0.8,0.85,0.9,0.95,1] }            
  
  
  print()
  
  grid_pnnstack = GridSearchCV(PNN(verbose=False), tuned_parameters, cv=10,
                         scoring='accuracy')
  t0=time()
  grid_pnnstack.fit(X_pred, y_train)
  t = time() - t0
  print("Best Hyper Parameters:\n",grid_pnnstack.best_params_)
  print(grid_pnnstack.best_score_)
  Time: 0:00:00 |N/A%|                                           | ETA:  --:--:--
  
  Best Hyper Parameters:
   {'std': 0.45}
  0.851528384279476
  In [30]:
  X_opt_pred=np.array([opt_svm_pred,opt_knn_pred,opt_RF_pred]).T
  #output layer containg y_test 
  y_test
  final_prediction=grid_pnnstack.best_estimator_.predict(X_opt_pred)
  print("=" * 52)
  print("time cost: {}".format(t))
  print()
  #importing the metrics module
  from sklearn import metrics
  #evaluation(Accuracy)
  print("Accuracy:",metrics.accuracy_score(final_prediction,y_test))
  #evaluation(Confusion Metrix)
  print("Confusion Metrix:\n",metrics.confusion_matrix(final_prediction,y_test))
  print()
  print("\t\taccuracy: {}".format(metrics.accuracy_score(final_prediction, y_test)))
  print(metrics.classification_report(final_prediction,y_test))
  ====================================================
  time cost: 3.23099684715271
  
  Accuracy: 0.8383838383838383
  Confusion Metrix:
   [[40  3]
   [13 43]]
  
          accuracy: 0.8383838383838383
               precision    recall  f1-score   support
  
            0       0.75      0.93      0.83        43
            1       0.93      0.77      0.84        56
  
  avg / total       0.86      0.84      0.84        99
  
  MLP Stacking¶
  In [30]:
  X_pred=np.array([svm_pred,knn_prediction,RF_prediction]).T
  #pred=np.array([pred1,pred2,pred3,pred4]).T
  #output layer containg y_test 
  y_train
  #split dataset
  from sklearn.neural_network import MLPClassifier
  from sklearn.model_selection import train_test_split
  from sklearn.model_selection import GridSearchCV
  from time import *
  np.random.seed(123)
  param_grid = {
          'activation' : ['identity', 'logistic', 'tanh', 'relu'],
      'solver' :['lbfgs', 'sgd', 'adam'] ,
      'alpha':[0.0001,1e-5,0.01,0.001],
      'batch_size' :[1000,5000],
      'learning_rate':['constant', 'invscaling', 'adaptive'] ,
      'learning_rate_init':[0.001,0.01,0.1,0.2,0.3] ,
      'hidden_layer_sizes':[[4,4],[4,5],[4,6],[4,7],[4,8],[4,9]]
  
  }
  mlp = MLPClassifier()
  grid_mlp = GridSearchCV(mlp, param_grid, cv=10, scoring='accuracy',n_jobs=-1)
  t0=time()
  grid_mlp.fit(X_pred, y_train)
  t = time() - t0
  print("Best Hyper Parameters:\n",grid_mlp.best_params_)
  print(grid_mlp.best_score_)
  Best Hyper Parameters:
   {'activation': 'identity', 'alpha': 0.01, 'batch_size': 5000, 'hidden_layer_sizes': [4, 8], 'learning_rate': 'constant', 'learning_rate_init': 0.1, 'solver': 'adam'}
  0.8646288209606987
  C:\Users\ixuser\Anaconda3\lib\site-packages\sklearn\neural_network\multilayer_perceptron.py:358: UserWarning: Got `batch_size` less than 1 or larger than sample size. It is going to be clipped
    warnings.warn("Got `batch_size` less than 1 or larger than "
  In [31]:
  X_opt_pred=np.array([opt_svm_pred,opt_knn_pred,opt_RF_pred]).T
  #output layer containg y_test 
  y_test
  final_prediction=grid_mlp.best_estimator_.predict(X_opt_pred)
  print("=" * 52)
  print("time cost: {}".format(t))
  print()
  #importing the metrics module
  from sklearn import metrics
  #evaluation(Accuracy)
  print("Accuracy:",metrics.accuracy_score(final_prediction,y_test))
  #evaluation(Confusion Metrix)
  print("Confusion Metrix:\n",metrics.confusion_matrix(final_prediction,y_test))
  print()
  print("\t\taccuracy: {}".format(metrics.accuracy_score(final_prediction, y_test)))
  print(metrics.classification_report(final_prediction,y_test))
  ====================================================
  time cost: 1520.9758558273315
  
  Accuracy: 0.8383838383838383
  Confusion Metrix:
   [[40  3]
   [13 43]]
  
          accuracy: 0.8383838383838383
               precision    recall  f1-score   support
  
            0       0.75      0.93      0.83        43
            1       0.93      0.77      0.84        56
  
  avg / total       0.86      0.84      0.84        99
  
  SVM Stacking¶
  In [31]:
  X_pred=np.array([svm_pred,knn_prediction,RF_prediction]).T
  #pred=np.array([pred1,pred2,pred3,pred4]).T
  #output layer containg y_test 
  y_train
  #split dataset
  from sklearn.neural_network import MLPClassifier
  from sklearn.model_selection import train_test_split
  from sklearn.model_selection import GridSearchCV
  from time import *
  np.random.seed(123)
  # Set the parameters by cross-validation
  tuned_parameters =  [{'kernel': ['rbf'], 'gamma': [1e-2, 1e-3, 1e-4, 1e-5],
                       'C': [0.001, 0.10, 0.1, 10, 25, 50, 100, 1000]},
                      {'kernel': ['sigmoid'], 'gamma': [1e-2, 1e-3, 1e-4, 1e-5],
                       'C': [0.001, 0.10, 0.1, 10, 25, 50, 100, 1000] },{'kernel': ['linear'], 'C': [0.001, 0.10, 0.1, 10, 25, 50, 100, 1000]}]              
  
  
  
  
  
  print()
  
  grid_sv = GridSearchCV(SVC(), tuned_parameters, cv=10,
                         scoring='accuracy')
  t0=time()
  grid_sv.fit(X_pred, y_train)
  t = time() - t0
  print("Best Hyper Parameters:\n",grid_sv.best_params_)
  print(grid_sv.best_score_)
  Best Hyper Parameters:
   {'C': 10, 'gamma': 0.01, 'kernel': 'rbf'}
  0.8558951965065502
  In [32]:
  X_opt_pred=np.array([opt_svm_pred,opt_knn_pred,opt_RF_pred]).T
  #output layer containg y_test 
  y_test
  final_prediction_sv=grid_sv.best_estimator_.predict(X_opt_pred)
  print("=" * 52)
  print("time cost: {}".format(t))
  print()
  #importing the metrics module
  from sklearn import metrics
  #evaluation(Accuracy)
  print("Accuracy:",metrics.accuracy_score(final_prediction_sv,y_test))
  #evaluation(Confusion Metrix)
  print("Confusion Metrix:\n",metrics.confusion_matrix(final_prediction_sv,y_test))
  print()
  print("\t\taccuracy: {}".format(metrics.accuracy_score(final_prediction_sv, y_test)))
  print(metrics.classification_report(final_prediction_sv,y_test))
  ====================================================
  time cost: 2.2583255767822266
  
  Accuracy: 0.8383838383838383
  Confusion Metrix:
   [[40  3]
   [13 43]]
  
          accuracy: 0.8383838383838383
               precision    recall  f1-score   support
  
            0       0.75      0.93      0.83        43
            1       0.93      0.77      0.84        56
  
  avg / total       0.86      0.84      0.84        99
  
  RF Stacking¶
  In [58]:
  X_pred=np.array([svm_pred,knn_prediction,RF_prediction]).T
  #pred=np.array([pred1,pred2,pred3,pred4]).T
  #output layer containg y_test 
  y_train
  #split dataset
  from sklearn.neural_network import MLPClassifier
  from sklearn.model_selection import train_test_split
  from sklearn.model_selection import GridSearchCV
  from time import *
  np.random.seed(123)
  modelS=RandomForestClassifier()
  #hyper parameters set
  params = {'criterion':['gini','entropy'],
            'n_estimators':[10,15,20,25,30],
            'min_samples_leaf':[1,2,3],
            'min_samples_split':[3,4,5,6,7],
            'random_state':[123],
            'n_jobs':[-1]}
  #Making models with hyper parameters sets
  RfgridS = GridSearchCV(modelS, param_grid=params,cv=10, n_jobs=-1)
  t0=time()
  RfgridS.fit(X_pred, y_train)
  t = time() - t0
  print("Best Hyper Parameters:\n",RfgridS.best_params_)
  print(RfgridS.best_score_)
  Best Hyper Parameters:
   {'criterion': 'gini', 'min_samples_leaf': 2, 'min_samples_split': 3, 'n_estimators': 10, 'n_jobs': -1, 'random_state': 123}
  0.851528384279476
  In [39]:
  X_opt_pred=np.array([opt_svm_pred,opt_knn_pred,opt_RF_pred]).T
  #output layer containg y_test 
  y_test
  final_prediction_rf=RfgridS.best_estimator_.predict(X_opt_pred)
  print("=" * 52)
  print("time cost: {}".format(t))
  print()
  #importing the metrics module
  from sklearn import metrics
  #evaluation(Accuracy)
  print("Accuracy:",metrics.accuracy_score(final_prediction_rf,y_test))
  #evaluation(Confusion Metrix)
  print("Confusion Metrix:\n",metrics.confusion_matrix(final_prediction_rf,y_test))
  print()
  print("\t\taccuracy: {}".format(metrics.accuracy_score(final_prediction_rf, y_test)))
  print(metrics.classification_report(final_prediction_rf,y_test))
  ====================================================
  time cost: 149.77435994148254
  
  Accuracy: 0.8383838383838383
  Confusion Metrix:
   [[40  3]
   [13 43]]
  
          accuracy: 0.8383838383838383
               precision    recall  f1-score   support
  
            0       0.75      0.93      0.83        43
            1       0.93      0.77      0.84        56
  
  avg / total       0.86      0.84      0.84        99
  X_pred=np.array([svm_pred,knn_prediction,RF_prediction]).T
  #pred=np.array([pred1,pred2,pred3,pred4]).T
  #output layer containg y_test 
  y_train
  #split dataset
  from sklearn.neural_network import MLPClassifier
  from sklearn.model_selection import train_test_split
  from sklearn.model_selection import GridSearchCV
  from time import *
  np.random.seed(123)
  modelS=RandomForestClassifier()
  #hyper parameters set
  params = {'criterion':['gini','entropy'],
            'n_estimators':[10,15,20,25,30],
            'min_samples_leaf':[1,2,3],
            'min_samples_split':[3,4,5,6,7],
            'random_state':[123],
            'n_jobs':[-1]}
  #Making models with hyper parameters sets
  RfgridS = GridSearchCV(modelS, param_grid=params,cv=10, n_jobs=-1)
  t0=time()
  RfgridS.fit(X_pred, y_train)
  t = time() - t0
  print("Best Hyper Parameters:\n",RfgridS.best_params_)
  print(RfgridS.best_score_)
  
  
  X_opt_pred=np.array([opt_svm_pred,opt_knn_pred,opt_RF_pred]).T
  #output layer containg y_test 
  y_test
  final_prediction_rf=RfgridS.best_estimator_.predict(X_opt_pred)
  print("=" * 52)
  print("time cost: {}".format(t))
  print()
  #importing the metrics module
  from sklearn import metrics
  #evaluation(Accuracy)
  print("Accuracy:",metrics.accuracy_score(final_prediction_rf,y_test))
  #evaluation(Confusion Metrix)
  print("Confusion Metrix:\n",metrics.confusion_matrix(final_prediction_rf,y_test))
  print()
  print("\t\taccuracy: {}".format(metrics.accuracy_score(final_prediction_rf, y_test)))
  print(metrics.classification_report(final_prediction_rf,y_test))
  
• Python: How import module as class?

  I had a problem when I wanted to create my factory.

  I have a directory that looks like that:

  classes/
       file_1.py
       file_2.py
       ....
  factory.py
  

  In file_1 and file_2, i have some classes definition in like this:

  class A():
      ...
  
  class B():
      ...
  

  In factory.py i want create a class on which I could use getattr function.

  I tried to do that:

  from classes import file_1, file_2
  
  class Factory(file_1, file_2):
      """doctring"""
  
  def build(dic):
      factory = Factory()
      return getattr(dic['name'], factory)(**dic['params'])
  

  But file_1 and file_2 are module, not a class.

  So what are the other ways that I can use to make my factory ?
  And if there are several, which is the best ?

  Thank you.
• Set message sender when sending SMS using AWS

  I'm using AWS SNS in order to send SMS to specific phone using the code below, every thing is working like a charm but I have one problem- the sender/message title is "NOTICE" and I wish to change it value. I didn't find any attribute that I can edit within the boto API to solve this issue...

  Any ideas ?? THANKS!!!

  import boto3
  sns = boto3.client('sns')
  sns.publish(
      PhoneNumber = '+11234567890',
      Message = 'Simple text message'
  )
  
• Running headless Firefox or chrome in DigitalOcean

  I am trying to run headless Chrome or Firefox in DigitalOcean and have tried alot of solutions but none seem to work.

  The code works superb in my local system but in my DigitalOcean server it doesn't.

  This is a test i am using to test headless Firefox

  from pyvirtualdisplay import Display
  from selenium import webdriver
  from selenium.webdriver.firefox.options import Options
  
  
  # print("open virtual display")
  with Display():
      print("open Firefox browser")
      browser = webdriver.Firefox()
      browser.set_window_size(1120, 550)
  
      url = 'http://arbspiper.com/'
      browser.get(url)
  
      title = browser.title
      print(title)
      browser.quit()
  

  The error i get is

  Traceback (most recent call last):
      File "firefox.py", line 9, in <module>
        browser = webdriver.Firefox()
      File "/home/arbspiper_project/env/lib/python3.5/site-packages/selenium/webdriver/firefox/webdriver.py", line 174, in __init__
  keep_alive=True)
      File "/home/arbspiper_project/env/lib/python3.5/site-packages/selenium/webdriver/remote/webdriver.py", line 157, in __init__
  self.start_session(capabilities, browser_profile)
      File "/home/arbspiper_project/env/lib/python3.5/site-packages/selenium/webdriver/remote/webdriver.py", line 252, in start_session
  response = self.execute(Command.NEW_SESSION, parameters)
      File "/home/arbspiper_project/env/lib/python3.5/site-packages/selenium/webdriver/remote/webdriver.py", line 321, in execute
  self.error_handler.check_response(response)
      File "/home/arbspiper_project/env/lib/python3.5/site-packages/selenium/webdriver/remote/errorhandler.py", line 242, in check_response
  raise exception_class(message, screen, stacktrace)
  
  selenium.common.exceptions.WebDriverException: Message: newSession
  

  Mozilla Firefox 64.0 Selenium version (3.141.0)
• how to navigate to new Url while opening new tab in IE

  I want to open 4 different urls in 4 different tabs in ie.. I used robot class to open tabs, I pressed ctrl +tab under that I have given url. Like that I have used 4 times ctrl+t. My code is working to open tabs. But url is opening in the Default window..
• Stale element reference: element is not attached to the page document for Java-Selenium

  What I am trying to do is after logging in, driver will click on Property dropdown, select an option and click on submit and repeat the process until the loop is complete.

  Below is my code:

  package com.genericlibrary;
  
  import java.util.List;
  
  import org.openqa.selenium.By;
  import org.openqa.selenium.WebDriver;
  import org.openqa.selenium.WebElement;
  import org.openqa.selenium.chrome.ChromeDriver;
  import org.openqa.selenium.support.ui.ExpectedConditions;
  import org.openqa.selenium.support.ui.Select;
  import org.openqa.selenium.support.ui.WebDriverWait;
  
  import com.util.Highlighter;
  
  public class MobopsSearchJobsFromDropDown {
  
      WebDriver driver;
      Highlighter color;
  
      public void getSetup() {
          String path = System.getProperty("user.dir");
          String driverPath = path + "\\Driver\\chromedriver.exe";
          System.setProperty("webdriver.chrome.driver", driverPath);
          driver = new ChromeDriver();
          driver.navigate().to("http://mobops-test.jcdecauxna.com/");
          driver.manage().window().maximize();
      }
  
      public void logIntoMobops() {
          WebElement userName = driver.findElement(By.xpath("//*[contains(@id,'username')]"));
          WebElement passWord = driver.findElement(By.xpath("//*[contains(@id,'password')]"));
          WebElement loginButton = driver.findElement(By.xpath("//*[contains(text(),'Login')]"));
  
          userName.sendKeys("test2");
          passWord.sendKeys("1234");
          loginButton.click();
  
      }
  
      public void selectEachPropertyAndSeachJob() {
          WebElement dateRange = driver.findElement(By.xpath("//*[contains(@name,'date_range')]"));
          WebElement last7days = driver.findElement(By.xpath("(//*[contains(text(),'Last 7 Days')])[2]"));
          WebElement searchJobs = driver.findElement(By.xpath("//*[contains(@name,'layout')]"));
          WebElement propertyDropdown = driver.findElement(By.xpath("//*[contains(@id,'property_id')]"));
  
          Select dropdown = new Select(propertyDropdown);
  
          List<WebElement> optionsInPropertyDropdown = dropdown.getOptions();
          for (int i = 0; i < optionsInPropertyDropdown.size(); i++) {
              if (propertyDropdown.isDisplayed() && propertyDropdown.isEnabled()) {
                  try {
                      propertyDropdown.click();
                      dropdown.selectByVisibleText(optionsInPropertyDropdown.get(i).getText());
                      dateRange.click();
                      last7days.click();
                      searchJobs.click();
                      System.out.println("Option Search is " + optionsInPropertyDropdown.get(i).getText());
  
                  } catch (org.openqa.selenium.StaleElementReferenceException ex) {
                      WebDriverWait wait = new WebDriverWait(driver, 30);
                      wait.until(ExpectedConditions.visibilityOf(propertyDropdown));
                  }
              }
  
          }
  
      }
  
      public static void main(String[] args) {
  
          MobopsSearchJobsFromDropDown obj = new MobopsSearchJobsFromDropDown();
          obj.getSetup();
          obj.logIntoMobops();
          obj.selectEachPropertyAndSeachJob();
  
      }
  }
  

  Right now it driver just select the first option and clicks on submit. Right after the page loads after the search completes I get the following error:

  Exception in thread "main" org.openqa.selenium.StaleElementReferenceException: stale element reference: element is not attached to the page document
  

  I have tried to implement the following code in my code, but since I am a novice, I have no idea how to implement the following to fix the issue:

  new WebDriverWait(driver, timeout)
          .ignoring(StaleElementReferenceException.class)
          .until((WebDriver d) -> {
              d.findElement(By.xpath("//*[contains(@id,'property_id')]")).click();
              return true;
          });
  

  Any help to overcome this issue is greatly appreciated. Thanks.
• Browser is not launching when jenkins running as a service?

  I have written test cases in selenium webdriver and testng. I am running Jenkins from command prompt and triggered a build it works perfectly. I have distributed tests in 3 VMs using selenium grid. I have converted JENKINS as a service. When running master/slave concept it is working fine. but I wanted to run in 3 to 4 VMs. So I am not going in master/ slave concept. So without master slave concept if I am running as Jenkins as a service browser is not launching.i have configured in Jenkins service as local system account administration allow interact with desktop but still it is not launching browser.
• ChromeOptions for converting to mobileweb is not working for RemoteWebDriver

  Chromeoptions for converting to mobileweb view is not working when RemoteWebDriver is used for grid. The code snippet is :

      DesiredCapabilities capabilities = DesiredCapabilities.chrome();
  
      Map<String, String> mobileEmulation = new HashMap<String, String>();
      mobileEmulation.put("deviceName", "iPhone 6");
      Map<String, Object> mobileOptions = new HashMap<String, Object>();
      mobileOptions.put("mobileEmulation", mobileEmulation);
  
      capabilities.setCapability(ChromeOptions.CAPABILITY, mobileOptions);
      if (host.equals("grid")) {
          driver = new RemoteWebDriver("http://localhost:4444/wd/hub", capabilities);
      } else if (host.equals("localhost")) {
          driver = new ChromeDriver(capabilities);
      }
  

  When host is used as localhost, the browser opens up in mobile view itself. But the problem arises when we try to run it in grid and pass the host as 'grid'. The browser does not open up in mobile view. Do we have a work around for this?

  Also, is it good to automate mobile web using Selenium, or I should move to appium for automating mobile web. I tried to start with selenium as it is much more easy to work with selenium than appium.

  Chrome Version - 70; Chrome Driver - 2.43; Selenium - 3.141.5

  The grid set up I am using is the official selenium grid
• Curl cmd and curl lib not yielding the same results

  im using curl lib to perform some webscraping on a swedish recipe database page(i have of course asked for permission). However im getting different results when i run curl in the commandline and curl lib with the same options.

  curl "https://kokboken.ikv.uu.se/sok.php" -H "User-Agent: Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:63.0) Gecko/20100101 Firefox/63.0" -H "Accept: text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8" -H "Accept-Language: sv-SE,sv;q=0.8,en-US;q=0.5,en;q=0.3" --compressed -H "Referer: https://kokboken.ikv.uu.se/sok.php" -H "Content-Type: application/x-www-form-urlencoded" -H "Connection: keep-alive" -H "Upgrade-Insecure-Requests: 1" --data "search_text=tomat&dummy=&search_type=all&rec_cats"%"5B"%"5D=all&submit_search=S"%"F6k&recid=&offset=0&searchid="
  

  this curl command returns the correct response which is a html page with a list of recipes.

  but when i run it in with the help of curllib(i use a c# wrapper called CurlThin) It returns the blank search page without any results as i my parameters where not passed. The expected behavior if i would enter the page via a browser.

  C# code

  private CURLcode _global;
  private SafeEasyHandle _easy;
  private readonly string HEMMETS_ROOT = "https://kokboken.ikv.uu.se/";
  
  _global = CurlNative.Init();
          // curl_easy_init() to create easy handle.
          _easy = CurlNative.Easy.Init();
  
          CurlNative.Easy.SetOpt(_easy, CURLoption.CAINFO, CurlResources.CaBundlePath);
          CurlNative.Easy.SetOpt(_easy, CURLoption.USERAGENT, "Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:63.0) Gecko/20100101 Firefox/63.0");
  
          //Add headers to the request 
          var headers = CurlNative.Slist.Append(SafeSlistHandle.Null, "Accept: text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8");
          // Add one more value to existing HTTP header list.
          CurlNative.Slist.Append(headers, "Accept-Language: sv-SE,sv;q=0.8,en-US;q=0.5,en;q=0.3");
          CurlNative.Slist.Append(headers, "Referer: https://kokboken.ikv.uu.se/sok.php");
          CurlNative.Slist.Append(headers, "Content-Type: application/x-www-form-urlencoded");
          CurlNative.Slist.Append(headers, "Connection: keep-alive");
          CurlNative.Slist.Append(headers, "Upgrade-Insecure-Requests: 1");
  
          // Configure libcurl easy handle to send HTTP headers we configured.
          CurlNative.Easy.SetOpt(_easy, CURLoption.HTTPHEADER, headers.DangerousGetHandle());
  
                      CurlNative.Easy.SetOpt(_easy, CURLoption.URL, HEMMETS_ROOT + "sok.php");
  
          // This one has to be called before setting COPYPOSTFIELDS.
          CurlNative.Easy.SetOpt(_easy, CURLoption.POSTFIELDS, postData);
          CurlNative.Easy.SetOpt(_easy, CURLoption.POST, 1);
          CurlNative.Easy.SetOpt(_easy, CURLoption.SSL_VERIFYHOST, 0);
          CurlNative.Easy.SetOpt(_easy, CURLoption.SSL_VERIFYPEER, 0);
  
          var stream = new MemoryStream();
          CurlNative.Easy.SetOpt(_easy, CURLoption.WRITEFUNCTION, (data, size, nmemb, user) =>
          {
              var length = (int)size * (int)nmemb;
              var buffer = new byte[length];
              Marshal.Copy(data, buffer, 0, length);
              stream.Write(buffer, 0, length);
              return (UIntPtr)length;
          });
  
          var result = CurlNative.Easy.Perform(_easy);
          html = Encoding.GetEncoding(1252).GetString(stream.ToArray());
  
          return html;
  

  I Suspect A: that my request is not performing a post as it is behaving as i opened the site with a get request. Or B: That im missing some kind of curl option that the commandline specifies automatically.
• Web Scaping for the website which was secured by incapsula

  Hello Python Developers, I am in need to scrape the data from the website here. This was protected by incapsula. I am already done two approaches and also used the techniques which were given by StackOverflow users.

  APPROACH 1:

  from incapsula import IncapSession
  
  headers = {'Host': 'www.vignanam.org',
             'User-Agent': 'Mozilla/5.0 (Windows; U; Windows NT 6.1; en-US) AppleWebKit/534.10 (KHTML, like Gecko) Chrome/7.0.540.0 Safari/534.10',
             'Accept': 'text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8',
             'Accept-Language': 'en-US,en;q=0.5',
             'Accept-Encoding': 'gzip, deflate',
             'Connection': 'keep-alive',
             'Cookie': 'visid_incap_1642409=B+YoelHCSKKN5z/Phs0zXCsF9VsAAAAAQUIPAAAAAACXaWvcNDXdMzcOky/SvffB; incap_ses'
                       '_715_1642409=kyFvSyJuuBVpNuh+aTHsCSsF9VsAAAAAKV6TIWTPSZmb+mOZWeuNHA==',
             'Upgrade-Insecure-Requests': '1'}
  
  session = IncapSession()
  response = session.get('http://www.vignanam.org/index.htm#&panel1-1', headers=headers, bypass_crack=True)
  
  print response.text
  

  APPROACH 2:

  from mechanize import Browser
  from bs4 import BeautifulSoup
  
  browser = Browser()
  
  browser.open('https://www.incapsula.com/blog/how-incapsula-protects-against-data-leaks.html')
  
  print browser.response()
  
  soup = BeautifulSoup(browser.response().read(), features='html5lib')
  
  print soup
  

  Both approaches are producing the same results. RESULT/OUTPUT

  <html> 
  <head> 
  <META NAME="robots" CONTENT="noindex,nofollow"> 
  <script src="/_Incapsula_Resource SWJIYLWA=5074a744e2e3d891814e9a2dace20bd4,719d34d31c8e3a6e6fffd425f7e032f3"> </script> 
  <body> 
  </body>
  </html>
  

  How to break this and scrape the data from there? Need urgent help. Please, Provide me with an answer or any other ways. Is any other programming languages to overcome this, please just it too.
• Why does my method not build a proper XML query from user input?

  I'm currently trying to take user input from a text box and build an XML query with it.

  The query should search within the document for an element with a given attribute 'dataItemId' and return the innerText value. The two lines of code which are commented out within the try{} statement work as expected, returning the correct value, yet when I try to use the two lines above these and type 'Xabs' into my input box I am getting:

  Your resultant Query XPath: //*[dataItemId = 'Xabs'] No Items Found."

  It must be something to do with how the string is parsed but I'm stumped as to what.

  public void MazakButton_Click(object sender, EventArgs e)
      {
          string userInput = searchInput.Text;
          ResultBox.Items.Clear();
          string query = "No Query Found";
          string searchResult = "No Items Found";
  
          if (userInput.Length > 3)// If query paramater is long enough
          {                
              //string Query = "\"//*[dataItemId = '" + userInput + "']\""; // Build Attribite Query
                  query = "//*[dataItemId = '" + userInput + "']"; // Build Attribite Query
              XmlDocument MTData = MTFunctions.ScrapeXMLData(MazakSourceURL, false);
  
              try
              {
                  XmlNode target = MTData.SelectSingleNode(query);
                  searchResult = userInput + ": " + target.InnerText;
                  //XmlNode target = MTData.SelectSingleNode("//*[@dataItemId = 'Xabs']"); 
                  //searchResult = target.InnerText;
              }
              catch
              {
                  Console.WriteLine("!!!!!!!!!!!!!!! - XML SEARCH FAILED - !!!!!!!!!!!!!!!");
              }
          }
          else{searchResult = "Invalid request (Too short)";}
  
  
          searchInput.Focus();
          ResultBox.Items.Add("Your Resultant Query XPath: " + query);
          ResultBox.Items.Add(searchResult);
          searchInput.Text = string.Empty; // Clear searchInput 
      }
  
• Finding src from video tag in a div in BeautifulSoup

  I'm trying to find the src of a video tag with BeautifulSoup in python

  import requests
  from urllib.request import urlopen
  from bs4 import BeautifulSoup as BS
  
  url = '<some url>'
  client_id = {'Client-ID': '<some id>'}
  
  json_data = requests.get(url, headers=client_id).json()
  
  def download(some_url) :
      html_page = urlopen(some_url)
      soup = BS(html_page, "html.parser")
  
      link_to_vid = soup.find('video')['src']
  
      print(link_to_vid)
  
      # urllib.request.urlretrieve(video)
  
  
  for x in range(0, num_clips):
      resp_url = (json_data['data'][x]['url'])
      print (resp_url)
      download(resp_url)
  

  When running this script I get an output of

  *link from print(resp_url)*
  Traceback (most recent call last):
    File "script.py", line 28 in <module>
        download(resp_url)
    File "script.py", line 18 in download
        link_to_vid = soup.find('video')['src']
  TypeError: 'NoneType' object is not subscriptable
  

  It seems to me that this error is occurring because BeautifulSoup cannot find a video tag on the webpage. I tried printing the whole html page I got from BeautifulSoup and it looks like it doesn't get the whole webpage, at least not what I can see from Chrome devtools.

  Am I getting this error because the video is nested deep within divs? Why isn't BeautifulSoup getting the whole html page?
• Trouble Parsing HTML

  I currently have a large HTML file filled with directories that I have to parse and extract the names, addresses, zip-codes, phone numbers, alternative phone numbers, and etc, where I will then put the contents inside an array. Some of these directory listings (like the snippet shown below) are less challenging and straightforward to parse.

  <span style="font-family: b'CUQZKJ+GaramondPremrPro-LtDisp'; font-size:12px">Aletheia House
  <br>201 Finley Avenue West
  <br>Birmingham, AL 35204
  <br>(205) 324-6502x22
  <br>Intake: (205) 324-6502
  </span>
  

  Sample Array Output

  ["Aletheia House", "201 Finley Avenue West",
   "Birmingham, AL 35204", "(205) 324-6502x22", "Intake: (205) 324-6502"]
  

  ---

  However, with scenarios like the snippet below, I am currently facing issues figuring out a way to go about parsing a scenario using BeautifulSoup, Python and RegEx.

  <span style="font-family: b'CUQZKJ+GaramondPremrPro-LtDisp'; font-size:12px">Alcohol and Drug Abuse
  <br>Treatment Centers Inc/Pearson Hall
  <br>2701 Jefferson Avenue SW
  <br>Birmingham, AL 35211
  <br>(205) 923-6552x12
  <br>Intakes: (205) 923-6552x10 (205) 923-
  <br>6552x13
  </span>
  

  Sample Array Output:

  ["Alcohol and Drug Abuse Treatment Centers Inc/Pearson Hall", 
   "2701 Jefferson Avenue SW", "Birmingham, AL 35211", "(205) 923-6552x12",
   "Intakes: (205) 923-6552x10 (205) 923-6552x13"]
  

  Thank you!
• BeautifulSoup stripping whitespace

  I am working on a basic horoscope parser from a website. Below is my code:

  import requests
  from bs4 import BeautifulSoup as bs
  
  url = "https://www.astrospeak.com/horoscope/capricorn"
  
  response = requests.request("GET", url)
  
  soup = bs(response.text, 'html.parser')
  
  locater = soup.select("#sunsignPredictionDiv > div.fullDIV > div.lineHght18 > div")
  
  quote = locater[0].previousSibling
  

  This leaves me with the following <class 'bs4.element.NavigableString'>:

  "\n                      You are working towards yet another dream and as you pursue this vision there's no doubt in your mind that it will come to fruition. It's written in the stars! \n                      "
  

  I am struggling how I can use the BeautifulSoup stripped_strings generator on the bs4.element.NavigableString. What I would like to end up with is just the string You are working towards yet another dream and as you pursue this vision there's no doubt in your mind that it will come to fruition. It's written in the stars!