Fatal编程技术网
  • python/
  • Python 属性错误：'；非类型'；对象没有属性'_jvm-PySpark-UDF

Python 属性错误：'；非类型'；对象没有属性'_jvm-PySpark-UDF

python apache-spark pyspark

Python 属性错误：'；非类型'；对象没有属性'_jvm-PySpark-UDF,python,apache-spark,pyspark,user-defined-functions,databricks,Python,Apache Spark,Pyspark,User Defined Functions,Databricks,我有杂志订阅的数据以及它们创建的时间，还有一列包含与给定用户关联的所有订阅过期日期的数组： user_id created_date expiration_dates_for_user 202394 '2018-05-04' ['2019-1-03', '2018-10-06', '2018-07-05'] 202394 '2017-01-04' ['2019-1-0

我有杂志订阅的数据以及它们创建的时间，还有一列包含与给定用户关联的所有订阅过期日期的数组：

 user_id           created_date             expiration_dates_for_user
 202394           '2018-05-04'           ['2019-1-03', '2018-10-06', '2018-07-05']
 202394           '2017-01-04'           ['2019-1-03', '2018-10-06', '2018-07-05']
 202394           '2016-05-04'           ['2019-1-03', '2018-10-06', '2018-07-05']
我正在尝试创建一个新列，该列包含创建日期45天内的所有过期日期，如下所示：

 user_id       created_date             expiration_dates_for_user                near_expiration_dates 
 202394       '2018-05-04'           ['2019-1-03', '2018-10-06', '2020-07-05']   []
 202394       '2019-01-04'           ['2019-1-03', '2018-10-06', '2020-07-05']   ['2019-1-03']
 202394       '2016-05-04'           ['2019-1-03', '2018-10-06', '2020-07-05']   []
以下是我正在使用的代码：

def check_if_sub_connected(created_at, expiration_array):
  if not expiration_array:
    return []
  if created_at == None:
    return []
  else:
    close_to_array = []
    for i in expiration_array:
      if datediff(created_at, i) < 45:
        if created_at != i:
          if datediff(created_at, i) > -45:
            close_to_array.append(i)
    return close_to_array

check_if_sub_connected = udf(check_if_sub_connected, ArrayType(TimestampType()))
我有一个疯狂的错误：

AttributeError: 'NoneType' object has no attribute '_jvm'

at org.apache.spark.api.python.BasePythonRunner$ReaderIterator.handlePythonException(PythonRunner.scala:317)
at org.apache.spark.sql.execution.python.PythonUDFRunner$$anon$1.read(PythonUDFRunner.scala:83)
at org.apache.spark.sql.execution.python.PythonUDFRunner$$anon$1.read(PythonUDFRunner.scala:66)
at org.apache.spark.api.python.BasePythonRunner$ReaderIterator.hasNext(PythonRunner.scala:271)
at org.apache.spark.InterruptibleIterator.hasNext(InterruptibleIterator.scala:37)
at scala.collection.Iterator$$anon$12.hasNext(Iterator.scala:439)
at scala.collection.Iterator$$anon$11.hasNext(Iterator.scala:408)
at scala.collection.Iterator$$anon$11.hasNext(Iterator.scala:408)
at org.apache.spark.sql.catalyst.expressions.GeneratedClass$GeneratedIteratorForCodegenStage17.processNext(Unknown Source)
at org.apache.spark.sql.execution.BufferedRowIterator.hasNext(BufferedRowIterator.java:43)
at org.apache.spark.sql.execution.WholeStageCodegenExec$$anonfun$10$$anon$1.hasNext(WholeStageCodegenExec.scala:620)
at org.apache.spark.sql.execution.collect.UnsafeRowBatchUtils$.encodeUnsafeRows(UnsafeRowBatchUtils.scala:49)
at org.apache.spark.sql.execution.collect.Collector$$anonfun$2.apply(Collector.scala:126)
at org.apache.spark.sql.execution.collect.Collector$$anonfun$2.apply(Collector.scala:125)
at org.apache.spark.scheduler.ResultTask.runTask(ResultTask.scala:87)
at org.apache.spark.scheduler.Task.run(Task.scala:112)
at org.apache.spark.executor.Executor$TaskRunner.run(Executor.scala:384)
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149)
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
at java.lang.Thread.run(Thread.java:748)

Driver stacktrace:
at org.apache.spark.scheduler.DAGScheduler.org$apache$spark$scheduler$DAGScheduler$$failJobAndIndependentStages(DAGScheduler.scala:1747)
at org.apache.spark.scheduler.DAGScheduler$$anonfun$abortStage$1.apply(DAGScheduler.scala:1735)
at org.apache.spark.scheduler.DAGScheduler$$anonfun$abortStage$1.apply(DAGScheduler.scala:1734)
at scala.collection.mutable.ResizableArray$class.foreach(ResizableArray.scala:59)
at scala.collection.mutable.ArrayBuffer.foreach(ArrayBuffer.scala:48)
at org.apache.spark.scheduler.DAGScheduler.abortStage(DAGScheduler.scala:1734)
at org.apache.spark.scheduler.DAGScheduler$$anonfun$handleTaskSetFailed$1.apply(DAGScheduler.scala:962)
at org.apache.spark.scheduler.DAGScheduler$$anonfun$handleTaskSetFailed$1.apply(DAGScheduler.scala:962)
at scala.Option.foreach(Option.scala:257)
at org.apache.spark.scheduler.DAGScheduler.handleTaskSetFailed(DAGScheduler.scala:962)
at org.apache.spark.scheduler.DAGSchedulerEventProcessLoop.doOnReceive(DAGScheduler.scala:1970)
at org.apache.spark.scheduler.DAGSchedulerEventProcessLoop.onReceive(DAGScheduler.scala:1918)
at org.apache.spark.scheduler.DAGSchedulerEventProcessLoop.onReceive(DAGScheduler.scala:1906)
at org.apache.spark.util.EventLoop$$anon$1.run(EventLoop.scala:49)
at org.apache.spark.scheduler.DAGScheduler.runJob(DAGScheduler.scala:759)
at org.apache.spark.SparkContext.runJob(SparkContext.scala:2141)
at org.apache.spark.sql.execution.collect.Collector.runSparkJobs(Collector.scala:237)
at org.apache.spark.sql.execution.collect.Collector.collect(Collector.scala:247)
at org.apache.spark.sql.execution.collect.Collector$.collect(Collector.scala:64)
at org.apache.spark.sql.execution.collect.Collector$.collect(Collector.scala:70)
at org.apache.spark.sql.execution.ResultCacheManager.getOrComputeResult(ResultCacheManager.scala:497)
at org.apache.spark.sql.execution.CollectLimitExec.executeCollectResult(limit.scala:48)
at org.apache.spark.sql.Dataset.org$apache$spark$sql$Dataset$$collectResult(Dataset.scala:2775)
at org.apache.spark.sql.Dataset.org$apache$spark$sql$Dataset$$collectFromPlan(Dataset.scala:3350)
at org.apache.spark.sql.Dataset$$anonfun$head$1.apply(Dataset.scala:2504)
at org.apache.spark.sql.Dataset$$anonfun$head$1.apply(Dataset.scala:2504)
at org.apache.spark.sql.Dataset$$anonfun$53.apply(Dataset.scala:3334)
at org.apache.spark.sql.execution.SQLExecution$$anonfun$withCustomExecutionEnv$1.apply(SQLExecution.scala:89)
at org.apache.spark.sql.execution.SQLExecution$.withSQLConfPropagated(SQLExecution.scala:175)
at org.apache.spark.sql.execution.SQLExecution$.withCustomExecutionEnv(SQLExecution.scala:84)
at org.apache.spark.sql.execution.SQLExecution$.withNewExecutionId(SQLExecution.scala:126)
at org.apache.spark.sql.Dataset.withAction(Dataset.scala:3333)
at org.apache.spark.sql.Dataset.head(Dataset.scala:2504)
at org.apache.spark.sql.Dataset.take(Dataset.scala:2718)
at org.apache.spark.sql.Dataset.showString(Dataset.scala:259)
at sun.reflect.GeneratedMethodAccessor472.invoke(Unknown Source)
at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
at java.lang.reflect.Method.invoke(Method.java:498)
at py4j.reflection.MethodInvoker.invoke(MethodInvoker.java:244)
at py4j.reflection.ReflectionEngine.invoke(ReflectionEngine.java:380)
at py4j.Gateway.invoke(Gateway.java:295)
at py4j.commands.AbstractCommand.invokeMethod(AbstractCommand.java:132)
at py4j.commands.CallCommand.execute(CallCommand.java:79)
at py4j.GatewayConnection.run(GatewayConnection.java:251)
at java.lang.Thread.run(Thread.java:748)
Caused by: org.apache.spark.api.python.PythonException: Traceback (most recent call last):
  File "/databricks/spark/python/pyspark/worker.py", line 262, in main
    process()
  File "/databricks/spark/python/pyspark/worker.py", line 257, in process
    serializer.dump_stream(func(split_index, iterator), outfile)
  File "/databricks/spark/python/pyspark/worker.py", line 183, in <lambda>
    func = lambda _, it: map(mapper, it)
  File "<string>", line 1, in <lambda>
  File "/databricks/spark/python/pyspark/worker.py", line 77, in <lambda>
    return lambda *a: toInternal(f(*a))
  File "/databricks/spark/python/pyspark/util.py", line 55, in wrapper
    return f(*args, **kwargs)
  File "<command-30583>", line 9, in check_if_sub_connected
  File "/databricks/spark/python/pyspark/sql/functions.py", line 1045, in datediff
    return Column(sc._jvm.functions.datediff(_to_java_column(end), _to_java_column(start)))
AttributeError: 'NoneType' object has no attribute '_jvm'
at org.apache.spark.api.python.BasePythonRunner$ReaderIterator.handlePythonException(PythonRunner.scala:317)
at org.apache.spark.sql.execution.python.PythonUDFRunner$$anon$1.read(PythonUDFRunner.scala:83)
at org.apache.spark.sql.execution.python.PythonUDFRunner$$anon$1.read(PythonUDFRunner.scala:66)
at org.apache.spark.api.python.BasePythonRunner$ReaderIterator.hasNext(PythonRunner.scala:271)
at org.apache.spark.InterruptibleIterator.hasNext(InterruptibleIterator.scala:37)
at scala.collection.Iterator$$anon$12.hasNext(Iterator.scala:439)
at scala.collection.Iterator$$anon$11.hasNext(Iterator.scala:408)
at scala.collection.Iterator$$anon$11.hasNext(Iterator.scala:408)
at org.apache.spark.sql.catalyst.expressions.GeneratedClass$GeneratedIteratorForCodegenStage17.processNext(Unknown Source)
at org.apache.spark.sql.execution.BufferedRowIterator.hasNext(BufferedRowIterator.java:43)
at org.apache.spark.sql.execution.WholeStageCodegenExec$$anonfun$10$$anon$1.hasNext(WholeStageCodegenExec.scala:620)
at org.apache.spark.sql.execution.collect.UnsafeRowBatchUtils$.encodeUnsafeRows(UnsafeRowBatchUtils.scala:49)
at org.apache.spark.sql.execution.collect.Collector$$anonfun$2.apply(Collector.scala:126)
at org.apache.spark.sql.execution.collect.Collector$$anonfun$2.apply(Collector.scala:125)
at org.apache.spark.scheduler.ResultTask.runTask(ResultTask.scala:87)
at org.apache.spark.scheduler.Task.run(Task.scala:112)
at org.apache.spark.executor.Executor$TaskRunner.run(Executor.scala:384)
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149)
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
... 1 more

AttributeError:'NoneType'对象没有属性'\u jvm'
位于org.apache.spark.api.python.BasePythonRunner$readeriator.handlePythonException（PythonRunner.scala:317）
位于org.apache.spark.sql.execution.python.PythonUDFRunner$$anon$1.read（PythonUDFRunner.scala:83）
位于org.apache.spark.sql.execution.python.PythonUDFRunner$$anon$1.read（PythonUDFRunner.scala:66）
位于org.apache.spark.api.python.BasePythonRunner$readerierator.hasNext（PythonRunner.scala:271）
在org.apache.spark.interruptblediator.hasNext（interruptblediator.scala:37）
位于scala.collection.Iterator$$anon$12.hasNext（Iterator.scala:439）
位于scala.collection.Iterator$$anon$11.hasNext（Iterator.scala:408）
位于scala.collection.Iterator$$anon$11.hasNext（Iterator.scala:408）
位于org.apache.spark.sql.catalyst.expressions.GeneratedClass$GenerateEditorForCodeGenStage17.processNext（未知源）
位于org.apache.spark.sql.execution.BufferedRowIterator.hasNext（BufferedRowIterator.java:43）
位于org.apache.spark.sql.execution.whisttagecodegenexec$$anonfun$10$$anon$1.hasNext（whisttagecodegenexec.scala:620）
位于org.apache.spark.sql.execution.collect.UnsafeRowBatchUtils$.encodeUnsafeRows（UnsafeRowBatchUtils.scala:49）
位于org.apache.spark.sql.execution.Collector.Collector$$anonfun$2.apply（Collector.scala:126）
位于org.apache.spark.sql.execution.Collector.Collector$$anonfun$2.apply（Collector.scala:125）
位于org.apache.spark.scheduler.ResultTask.runTask（ResultTask.scala:87）
位于org.apache.spark.scheduler.Task.run（Task.scala:112）
位于org.apache.spark.executor.executor$TaskRunner.run（executor.scala:384）
位于java.util.concurrent.ThreadPoolExecutor.runWorker（ThreadPoolExecutor.java:1149）
位于java.util.concurrent.ThreadPoolExecutor$Worker.run（ThreadPoolExecutor.java:624）
运行（Thread.java:748）
驱动程序堆栈跟踪：
位于org.apache.spark.scheduler.DAGScheduler.org$apache$spark$scheduler$DAGScheduler$$failJobAndIndependentStages（DAGScheduler.scala:1747）
位于org.apache.spark.scheduler.DAGScheduler$$anonfun$abortStage$1.apply（DAGScheduler.scala:1735）
位于org.apache.spark.scheduler.DAGScheduler$$anonfun$abortStage$1.apply（DAGScheduler.scala:1734）
位于scala.collection.mutable.resizeblearray$class.foreach（resizeblearray.scala:59）
位于scala.collection.mutable.ArrayBuffer.foreach（ArrayBuffer.scala:48）
位于org.apache.spark.scheduler.DAGScheduler.abortStage（DAGScheduler.scala:1734）
位于org.apache.spark.scheduler.DAGScheduler$$anonfun$handleTaskSetFailed$1.apply（DAGScheduler.scala:962）
位于org.apache.spark.scheduler.DAGScheduler$$anonfun$handleTaskSetFailed$1.apply（DAGScheduler.scala:962）
位于scala.Option.foreach（Option.scala:257）
位于org.apache.spark.scheduler.DAGScheduler.handleTaskSetFailed（DAGScheduler.scala:962）
位于org.apache.spark.scheduler.DAGSchedulerEventProcessLoop.doOnReceive（DAGScheduler.scala:1970）
位于org.apache.spark.scheduler.DAGSchedulerEventProcessLoop.onReceive（DAGScheduler.scala:1918）
位于org.apache.spark.scheduler.DAGSchedulerEventProcessLoop.onReceive（DAGScheduler.scala:1906）
位于org.apache.spark.util.EventLoop$$anon$1.run（EventLoop.scala:49）
位于org.apache.spark.scheduler.DAGScheduler.runJob（DAGScheduler.scala:759）
位于org.apache.spark.SparkContext.runJob（SparkContext.scala:2141）
位于org.apache.spark.sql.execution.collect.Collector.runSparkJobs（Collector.scala:237）
位于org.apache.spark.sql.execution.Collector.collect（Collector.scala:247）
位于org.apache.spark.sql.execution.Collector$.collect（Collector.scala:64）
位于org.apache.spark.sql.execution.Collector$.collect（Collector.scala:70）
位于org.apache.spark.sql.execution.ResultCacheManager.getOrComputeResult（ResultCacheManager.scala:497）
位于org.apache.spark.sql.execution.CollectLimitExec.executeCollectResult（limit.scala:48）
位于org.apache.spark.sql.Dataset.org$apache$spark$sql$Dataset$$collectResult（Dataset.scala:2775）
位于org.apache.spark.sql.Dataset.org$apache$spark$sql$Dataset$$collectFromPlan（Dataset.scala:3350）
位于org.apache.spark.sql.Dataset$$anonfun$head$1.apply（Dataset.scala:2504）
位于org.apache.spark.sql.Dataset$$anonfun$head$1.apply（Dataset.scala:2504）
位于org.apache.spark.sql.Dataset$$anonfun$53.apply（Dataset.scala:3334）
位于org.apache.spark.sql.execution.SQLExecution$$anonfun$和customexecutionenv$1.apply（SQLExecution.scala:89）
在org.apache.spark.sql.execution.SQLExecution$.withSQLConfPropagated（SQLExecution.scala:175）
位于org.apache.spark.sql.execution.SQLExecution$.withCustomExecutionEnv（SQLExecution.scala:84）
位于org.apache.spark.sql.execution.SQLExecution$.withNewExecutionId（SQLExecution.scala:126）
位于org.apache.spark.sql.Dataset.withAction（Dataset.scala:3333）
位于org.apache.spark.sql.Dataset.head（Dataset.scala:2504）
位于org.apache.spark.sql.Dataset.take（Dataset.scala:2718）
位于org.apache.spark.sql.Dataset.showString（Dataset.scala:259）
位于sun.reflect.GeneratedMethodAccessor472.invoke（未知源）
在sun.reflect.DelegatingMethodAccessorImpl.invoke（DelegatingMethodAccessorImpl.java:43）中
位于java.lang.reflect.Method.invoke（Method.java:498）
位于py4j.reflection.MethodInvoker.invoke（MethodInvoker.java:244）
位于py4j.reflection.ReflectionEngine.invoke（ReflectionEngine.java:380）
在py4j.Gateway.invoke处（Gateway.java:295）
位于py4j.commands.AbstractCommand.invokeMethod（AbstractCommand.java:132）
在py4j.commands.CallCommand.execute（CallCommand.java:79）
在py4j.GatewayConnection.run处（GatewayConnection.java:251）
运行（Thread.java:748）
原因：org.apache.spark.api.python.python异常：回溯（最近一次调用last）：
文件“/databricks/spark/python/pyspark/worker.py”，主文件第262行
过程（）
文件“/数据

AttributeError: 'NoneType' object has no attribute '_jvm'

at org.apache.spark.api.python.BasePythonRunner$ReaderIterator.handlePythonException(PythonRunner.scala:317)
at org.apache.spark.sql.execution.python.PythonUDFRunner$$anon$1.read(PythonUDFRunner.scala:83)
at org.apache.spark.sql.execution.python.PythonUDFRunner$$anon$1.read(PythonUDFRunner.scala:66)
at org.apache.spark.api.python.BasePythonRunner$ReaderIterator.hasNext(PythonRunner.scala:271)
at org.apache.spark.InterruptibleIterator.hasNext(InterruptibleIterator.scala:37)
at scala.collection.Iterator$$anon$12.hasNext(Iterator.scala:439)
at scala.collection.Iterator$$anon$11.hasNext(Iterator.scala:408)
at scala.collection.Iterator$$anon$11.hasNext(Iterator.scala:408)
at org.apache.spark.sql.catalyst.expressions.GeneratedClass$GeneratedIteratorForCodegenStage17.processNext(Unknown Source)
at org.apache.spark.sql.execution.BufferedRowIterator.hasNext(BufferedRowIterator.java:43)
at org.apache.spark.sql.execution.WholeStageCodegenExec$$anonfun$10$$anon$1.hasNext(WholeStageCodegenExec.scala:620)
at org.apache.spark.sql.execution.collect.UnsafeRowBatchUtils$.encodeUnsafeRows(UnsafeRowBatchUtils.scala:49)
at org.apache.spark.sql.execution.collect.Collector$$anonfun$2.apply(Collector.scala:126)
at org.apache.spark.sql.execution.collect.Collector$$anonfun$2.apply(Collector.scala:125)
at org.apache.spark.scheduler.ResultTask.runTask(ResultTask.scala:87)
at org.apache.spark.scheduler.Task.run(Task.scala:112)
at org.apache.spark.executor.Executor$TaskRunner.run(Executor.scala:384)
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149)
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
at java.lang.Thread.run(Thread.java:748)

Driver stacktrace:
at org.apache.spark.scheduler.DAGScheduler.org$apache$spark$scheduler$DAGScheduler$$failJobAndIndependentStages(DAGScheduler.scala:1747)
at org.apache.spark.scheduler.DAGScheduler$$anonfun$abortStage$1.apply(DAGScheduler.scala:1735)
at org.apache.spark.scheduler.DAGScheduler$$anonfun$abortStage$1.apply(DAGScheduler.scala:1734)
at scala.collection.mutable.ResizableArray$class.foreach(ResizableArray.scala:59)
at scala.collection.mutable.ArrayBuffer.foreach(ArrayBuffer.scala:48)
at org.apache.spark.scheduler.DAGScheduler.abortStage(DAGScheduler.scala:1734)
at org.apache.spark.scheduler.DAGScheduler$$anonfun$handleTaskSetFailed$1.apply(DAGScheduler.scala:962)
at org.apache.spark.scheduler.DAGScheduler$$anonfun$handleTaskSetFailed$1.apply(DAGScheduler.scala:962)
at scala.Option.foreach(Option.scala:257)
at org.apache.spark.scheduler.DAGScheduler.handleTaskSetFailed(DAGScheduler.scala:962)
at org.apache.spark.scheduler.DAGSchedulerEventProcessLoop.doOnReceive(DAGScheduler.scala:1970)
at org.apache.spark.scheduler.DAGSchedulerEventProcessLoop.onReceive(DAGScheduler.scala:1918)
at org.apache.spark.scheduler.DAGSchedulerEventProcessLoop.onReceive(DAGScheduler.scala:1906)
at org.apache.spark.util.EventLoop$$anon$1.run(EventLoop.scala:49)
at org.apache.spark.scheduler.DAGScheduler.runJob(DAGScheduler.scala:759)
at org.apache.spark.SparkContext.runJob(SparkContext.scala:2141)
at org.apache.spark.sql.execution.collect.Collector.runSparkJobs(Collector.scala:237)
at org.apache.spark.sql.execution.collect.Collector.collect(Collector.scala:247)
at org.apache.spark.sql.execution.collect.Collector$.collect(Collector.scala:64)
at org.apache.spark.sql.execution.collect.Collector$.collect(Collector.scala:70)
at org.apache.spark.sql.execution.ResultCacheManager.getOrComputeResult(ResultCacheManager.scala:497)
at org.apache.spark.sql.execution.CollectLimitExec.executeCollectResult(limit.scala:48)
at org.apache.spark.sql.Dataset.org$apache$spark$sql$Dataset$$collectResult(Dataset.scala:2775)
at org.apache.spark.sql.Dataset.org$apache$spark$sql$Dataset$$collectFromPlan(Dataset.scala:3350)
at org.apache.spark.sql.Dataset$$anonfun$head$1.apply(Dataset.scala:2504)
at org.apache.spark.sql.Dataset$$anonfun$head$1.apply(Dataset.scala:2504)
at org.apache.spark.sql.Dataset$$anonfun$53.apply(Dataset.scala:3334)
at org.apache.spark.sql.execution.SQLExecution$$anonfun$withCustomExecutionEnv$1.apply(SQLExecution.scala:89)
at org.apache.spark.sql.execution.SQLExecution$.withSQLConfPropagated(SQLExecution.scala:175)
at org.apache.spark.sql.execution.SQLExecution$.withCustomExecutionEnv(SQLExecution.scala:84)
at org.apache.spark.sql.execution.SQLExecution$.withNewExecutionId(SQLExecution.scala:126)
at org.apache.spark.sql.Dataset.withAction(Dataset.scala:3333)
at org.apache.spark.sql.Dataset.head(Dataset.scala:2504)
at org.apache.spark.sql.Dataset.take(Dataset.scala:2718)
at org.apache.spark.sql.Dataset.showString(Dataset.scala:259)
at sun.reflect.GeneratedMethodAccessor472.invoke(Unknown Source)
at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
at java.lang.reflect.Method.invoke(Method.java:498)
at py4j.reflection.MethodInvoker.invoke(MethodInvoker.java:244)
at py4j.reflection.ReflectionEngine.invoke(ReflectionEngine.java:380)
at py4j.Gateway.invoke(Gateway.java:295)
at py4j.commands.AbstractCommand.invokeMethod(AbstractCommand.java:132)
at py4j.commands.CallCommand.execute(CallCommand.java:79)
at py4j.GatewayConnection.run(GatewayConnection.java:251)
at java.lang.Thread.run(Thread.java:748)
Caused by: org.apache.spark.api.python.PythonException: Traceback (most recent call last):
  File "/databricks/spark/python/pyspark/worker.py", line 262, in main
    process()
  File "/databricks/spark/python/pyspark/worker.py", line 257, in process
    serializer.dump_stream(func(split_index, iterator), outfile)
  File "/databricks/spark/python/pyspark/worker.py", line 183, in <lambda>
    func = lambda _, it: map(mapper, it)
  File "<string>", line 1, in <lambda>
  File "/databricks/spark/python/pyspark/worker.py", line 77, in <lambda>
    return lambda *a: toInternal(f(*a))
  File "/databricks/spark/python/pyspark/util.py", line 55, in wrapper
    return f(*args, **kwargs)
  File "<command-30583>", line 9, in check_if_sub_connected
  File "/databricks/spark/python/pyspark/sql/functions.py", line 1045, in datediff
    return Column(sc._jvm.functions.datediff(_to_java_column(end), _to_java_column(start)))
AttributeError: 'NoneType' object has no attribute '_jvm'
at org.apache.spark.api.python.BasePythonRunner$ReaderIterator.handlePythonException(PythonRunner.scala:317)
at org.apache.spark.sql.execution.python.PythonUDFRunner$$anon$1.read(PythonUDFRunner.scala:83)
at org.apache.spark.sql.execution.python.PythonUDFRunner$$anon$1.read(PythonUDFRunner.scala:66)
at org.apache.spark.api.python.BasePythonRunner$ReaderIterator.hasNext(PythonRunner.scala:271)
at org.apache.spark.InterruptibleIterator.hasNext(InterruptibleIterator.scala:37)
at scala.collection.Iterator$$anon$12.hasNext(Iterator.scala:439)
at scala.collection.Iterator$$anon$11.hasNext(Iterator.scala:408)
at scala.collection.Iterator$$anon$11.hasNext(Iterator.scala:408)
at org.apache.spark.sql.catalyst.expressions.GeneratedClass$GeneratedIteratorForCodegenStage17.processNext(Unknown Source)
at org.apache.spark.sql.execution.BufferedRowIterator.hasNext(BufferedRowIterator.java:43)
at org.apache.spark.sql.execution.WholeStageCodegenExec$$anonfun$10$$anon$1.hasNext(WholeStageCodegenExec.scala:620)
at org.apache.spark.sql.execution.collect.UnsafeRowBatchUtils$.encodeUnsafeRows(UnsafeRowBatchUtils.scala:49)
at org.apache.spark.sql.execution.collect.Collector$$anonfun$2.apply(Collector.scala:126)
at org.apache.spark.sql.execution.collect.Collector$$anonfun$2.apply(Collector.scala:125)
at org.apache.spark.scheduler.ResultTask.runTask(ResultTask.scala:87)
at org.apache.spark.scheduler.Task.run(Task.scala:112)
at org.apache.spark.executor.Executor$TaskRunner.run(Executor.scala:384)
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149)
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
... 1 more



def check_if_sub_connected(created_at, expiration_array):
  if not expiration_array:
    return []
  else:
    close_to_array = []
    for i in expiration_array:
      if created_at - i < pd.Timedelta(days=45):
        if created_at - i > pd.Timedelta(days=-45):
          close_to_array.append(i)
    return close_to_array

check_if_sub_connected = udf(check_if_sub_connected, ArrayType(TimestampType()))